1f5166768STheodore Ts'o // SPDX-License-Identifier: GPL-2.0 2c9de560dSAlex Tomas /* 3c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 4c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 5c9de560dSAlex Tomas */ 6c9de560dSAlex Tomas 7c9de560dSAlex Tomas 8c9de560dSAlex Tomas /* 9c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 10c9de560dSAlex Tomas */ 11c9de560dSAlex Tomas 1218aadd47SBobi Jam #include "ext4_jbd2.h" 138f6e39a7SMingming Cao #include "mballoc.h" 1428623c2fSTheodore Ts'o #include <linux/log2.h> 15a0b30c12STheodore Ts'o #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 171a5d5e5dSJeremy Cline #include <linux/nospec.h> 1866114cadSTejun Heo #include <linux/backing-dev.h> 199bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 209bffad1eSTheodore Ts'o 21c9de560dSAlex Tomas /* 22c9de560dSAlex Tomas * MUSTDO: 23c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 24c9de560dSAlex Tomas * - search for metadata in few groups 25c9de560dSAlex Tomas * 26c9de560dSAlex Tomas * TODO v4: 27c9de560dSAlex Tomas * - normalization should take into account whether file is still open 28c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 29c9de560dSAlex Tomas * - don't normalize tails 30c9de560dSAlex Tomas * - quota 31c9de560dSAlex Tomas * - reservation for superuser 32c9de560dSAlex Tomas * 33c9de560dSAlex Tomas * TODO v3: 34c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 35c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 36c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 37c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 38c9de560dSAlex Tomas * - error handling 39c9de560dSAlex Tomas */ 40c9de560dSAlex Tomas 41c9de560dSAlex Tomas /* 42c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 43c9de560dSAlex Tomas * near to the goal(block) value specified. 44c9de560dSAlex Tomas * 45b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 46b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 47b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 48b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 49b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 50b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 51b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 52b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 53b713a5ecSTheodore Ts'o * terms of number of blocks. 54c9de560dSAlex Tomas * 55c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 56b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 57c9de560dSAlex Tomas * 58b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 59b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 60b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 61b713a5ecSTheodore Ts'o * represented as: 62c9de560dSAlex Tomas * 63c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 64c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 6553accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 6653accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 67c9de560dSAlex Tomas * 68c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 69c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 70caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 71caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 72c9de560dSAlex Tomas * 73c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 74c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 75c9de560dSAlex Tomas * pa_free. 76c9de560dSAlex Tomas * 77c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 78c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 79caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 80c9de560dSAlex Tomas * 81c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 82c9de560dSAlex Tomas * 83c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 84c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 85c9de560dSAlex Tomas * 86c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 8725985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 88c9de560dSAlex Tomas * 89c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 90c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 91c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 92c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 93c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 94c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 95c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 96c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 97c9de560dSAlex Tomas * inode as: 98c9de560dSAlex Tomas * 99c9de560dSAlex Tomas * { page } 100c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 101c9de560dSAlex Tomas * 102c9de560dSAlex Tomas * 103c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 104ea1754a0SKirill A. Shutemov * take up 2 blocks. A page can contain blocks_per_page (PAGE_SIZE / 105c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 106c9de560dSAlex Tomas * which is blocks_per_page/2 107c9de560dSAlex Tomas * 108c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 109c9de560dSAlex Tomas * away when the filesystem is unmounted. 110c9de560dSAlex Tomas * 111c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 112c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 113c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 114c9de560dSAlex Tomas * 115c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 116c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 117c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 118c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 119c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 120c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 12127baebb8STheodore Ts'o * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is 12227baebb8STheodore Ts'o * dependent on the cluster size; for non-bigalloc file systems, it is 123c9de560dSAlex Tomas * 512 blocks. This can be tuned via 124d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 125c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 126c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 127d7a1fee1SDan Ehrenberg * the smallest multiple of the stripe value (sbi->s_stripe) which is 128d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 129c9de560dSAlex Tomas * 130d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 131c9de560dSAlex Tomas * 132b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 133b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 134b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 135c9de560dSAlex Tomas * 136b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 137c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 138c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 139b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 140af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 141b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 142b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 143b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 144c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 145b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 146c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 147c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 148c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 149caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 150c9de560dSAlex Tomas * checked. 151c9de560dSAlex Tomas * 152c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 153c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 154c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 155c9de560dSAlex Tomas * subsequent request. 156c9de560dSAlex Tomas */ 157c9de560dSAlex Tomas 158c9de560dSAlex Tomas /* 159c9de560dSAlex Tomas * mballoc operates on the following data: 160c9de560dSAlex Tomas * - on-disk bitmap 161c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 162c9de560dSAlex Tomas * - preallocation descriptors (PAs) 163c9de560dSAlex Tomas * 164c9de560dSAlex Tomas * there are two types of preallocations: 165c9de560dSAlex Tomas * - inode 166c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 167c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 168c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 169c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 170c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 171c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 172c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 173c9de560dSAlex Tomas * must discard all preallocated blocks. 174c9de560dSAlex Tomas * - locality group 175c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 176c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 177c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 178c9de560dSAlex Tomas * it's consumed from the beginning to the end. 179c9de560dSAlex Tomas * 180c9de560dSAlex Tomas * relation between them can be expressed as: 181c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 182c9de560dSAlex Tomas * 183c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 184c9de560dSAlex Tomas * - allocated blocks (persistent) 185c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 186c9de560dSAlex Tomas * 187c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 188c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 189c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 190c9de560dSAlex Tomas * 191c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 192c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 193c9de560dSAlex Tomas * 194c9de560dSAlex Tomas * all operations can be expressed as: 195c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 196c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 197c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 198c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 199c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 200c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 201c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 202c9de560dSAlex Tomas * is used in real operation because we can't know actual used 203c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 204c9de560dSAlex Tomas * 205c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 206c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 207c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 208c9de560dSAlex Tomas * the following knowledge: 209c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 210c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 211c9de560dSAlex Tomas * nobody can re-allocate that block 212c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 213c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 214c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 215c9de560dSAlex Tomas * block 216c9de560dSAlex Tomas * 217c9de560dSAlex Tomas * so, now we're building a concurrency table: 218c9de560dSAlex Tomas * - init buddy vs. 219c9de560dSAlex Tomas * - new PA 220c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 221c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 222c9de560dSAlex Tomas * - use inode PA 223c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 224c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 225c9de560dSAlex Tomas * - discard inode PA 226c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 227c9de560dSAlex Tomas * - use locality group PA 228c9de560dSAlex Tomas * again PA-=N must be serialized with init 229c9de560dSAlex Tomas * - discard locality group PA 230c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 231c9de560dSAlex Tomas * - new PA vs. 232c9de560dSAlex Tomas * - use inode PA 233c9de560dSAlex Tomas * i_data_sem serializes them 234c9de560dSAlex Tomas * - discard inode PA 235c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 236c9de560dSAlex Tomas * - use locality group PA 237c9de560dSAlex Tomas * some mutex should serialize them 238c9de560dSAlex Tomas * - discard locality group PA 239c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 240c9de560dSAlex Tomas * - use inode PA 241c9de560dSAlex Tomas * - use inode PA 242c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 243c9de560dSAlex Tomas * - discard inode PA 244c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 245c9de560dSAlex Tomas * - use locality group PA 246c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 247c9de560dSAlex Tomas * - discard locality group PA 248c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 249c9de560dSAlex Tomas * 250c9de560dSAlex Tomas * now we're ready to make few consequences: 251c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 252c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 253c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 254c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 255c9de560dSAlex Tomas * any discard or they're serialized somehow 256c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 257c9de560dSAlex Tomas * 258c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 259c9de560dSAlex Tomas * in this case, but we should care about concurrent init 260c9de560dSAlex Tomas * 261c9de560dSAlex Tomas */ 262c9de560dSAlex Tomas 263c9de560dSAlex Tomas /* 264c9de560dSAlex Tomas * Logic in few words: 265c9de560dSAlex Tomas * 266c9de560dSAlex Tomas * - allocation: 267c9de560dSAlex Tomas * load group 268c9de560dSAlex Tomas * find blocks 269c9de560dSAlex Tomas * mark bits in on-disk bitmap 270c9de560dSAlex Tomas * release group 271c9de560dSAlex Tomas * 272c9de560dSAlex Tomas * - use preallocation: 273c9de560dSAlex Tomas * find proper PA (per-inode or group) 274c9de560dSAlex Tomas * load group 275c9de560dSAlex Tomas * mark bits in on-disk bitmap 276c9de560dSAlex Tomas * release group 277c9de560dSAlex Tomas * release PA 278c9de560dSAlex Tomas * 279c9de560dSAlex Tomas * - free: 280c9de560dSAlex Tomas * load group 281c9de560dSAlex Tomas * mark bits in on-disk bitmap 282c9de560dSAlex Tomas * release group 283c9de560dSAlex Tomas * 284c9de560dSAlex Tomas * - discard preallocations in group: 285c9de560dSAlex Tomas * mark PAs deleted 286c9de560dSAlex Tomas * move them onto local list 287c9de560dSAlex Tomas * load on-disk bitmap 288c9de560dSAlex Tomas * load group 289c9de560dSAlex Tomas * remove PA from object (inode or locality group) 290c9de560dSAlex Tomas * mark free blocks in-core 291c9de560dSAlex Tomas * 292c9de560dSAlex Tomas * - discard inode's preallocations: 293c9de560dSAlex Tomas */ 294c9de560dSAlex Tomas 295c9de560dSAlex Tomas /* 296c9de560dSAlex Tomas * Locking rules 297c9de560dSAlex Tomas * 298c9de560dSAlex Tomas * Locks: 299c9de560dSAlex Tomas * - bitlock on a group (group) 300c9de560dSAlex Tomas * - object (inode/locality) (object) 301c9de560dSAlex Tomas * - per-pa lock (pa) 302c9de560dSAlex Tomas * 303c9de560dSAlex Tomas * Paths: 304c9de560dSAlex Tomas * - new pa 305c9de560dSAlex Tomas * object 306c9de560dSAlex Tomas * group 307c9de560dSAlex Tomas * 308c9de560dSAlex Tomas * - find and use pa: 309c9de560dSAlex Tomas * pa 310c9de560dSAlex Tomas * 311c9de560dSAlex Tomas * - release consumed pa: 312c9de560dSAlex Tomas * pa 313c9de560dSAlex Tomas * group 314c9de560dSAlex Tomas * object 315c9de560dSAlex Tomas * 316c9de560dSAlex Tomas * - generate in-core bitmap: 317c9de560dSAlex Tomas * group 318c9de560dSAlex Tomas * pa 319c9de560dSAlex Tomas * 320c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 321c9de560dSAlex Tomas * object 322c9de560dSAlex Tomas * pa 323c9de560dSAlex Tomas * group 324c9de560dSAlex Tomas * 325c9de560dSAlex Tomas * - discard all for given group: 326c9de560dSAlex Tomas * group 327c9de560dSAlex Tomas * pa 328c9de560dSAlex Tomas * group 329c9de560dSAlex Tomas * object 330c9de560dSAlex Tomas * 331c9de560dSAlex Tomas */ 332c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 333c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 33418aadd47SBobi Jam static struct kmem_cache *ext4_free_data_cachep; 335fb1813f4SCurt Wohlgemuth 336fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 337fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 338fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 3392892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 340fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 341fb1813f4SCurt Wohlgemuth 342d6006186SEric Biggers static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 3432892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 3442892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 3452892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 3462892c15dSEric Sandeen }; 3472892c15dSEric Sandeen 348c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 349c3a326a6SAneesh Kumar K.V ext4_group_t group); 3507a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 3517a2fcbf7SAneesh Kumar K.V ext4_group_t group); 35253f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac); 353c3a326a6SAneesh Kumar K.V 35407b5b8e1SRitesh Harjani /* 35507b5b8e1SRitesh Harjani * The algorithm using this percpu seq counter goes below: 35607b5b8e1SRitesh Harjani * 1. We sample the percpu discard_pa_seq counter before trying for block 35707b5b8e1SRitesh Harjani * allocation in ext4_mb_new_blocks(). 35807b5b8e1SRitesh Harjani * 2. We increment this percpu discard_pa_seq counter when we either allocate 35907b5b8e1SRitesh Harjani * or free these blocks i.e. while marking those blocks as used/free in 36007b5b8e1SRitesh Harjani * mb_mark_used()/mb_free_blocks(). 36107b5b8e1SRitesh Harjani * 3. We also increment this percpu seq counter when we successfully identify 36207b5b8e1SRitesh Harjani * that the bb_prealloc_list is not empty and hence proceed for discarding 36307b5b8e1SRitesh Harjani * of those PAs inside ext4_mb_discard_group_preallocations(). 36407b5b8e1SRitesh Harjani * 36507b5b8e1SRitesh Harjani * Now to make sure that the regular fast path of block allocation is not 36607b5b8e1SRitesh Harjani * affected, as a small optimization we only sample the percpu seq counter 36707b5b8e1SRitesh Harjani * on that cpu. Only when the block allocation fails and when freed blocks 36807b5b8e1SRitesh Harjani * found were 0, that is when we sample percpu seq counter for all cpus using 36907b5b8e1SRitesh Harjani * below function ext4_get_discard_pa_seq_sum(). This happens after making 37007b5b8e1SRitesh Harjani * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty. 37107b5b8e1SRitesh Harjani */ 37207b5b8e1SRitesh Harjani static DEFINE_PER_CPU(u64, discard_pa_seq); 37307b5b8e1SRitesh Harjani static inline u64 ext4_get_discard_pa_seq_sum(void) 37407b5b8e1SRitesh Harjani { 37507b5b8e1SRitesh Harjani int __cpu; 37607b5b8e1SRitesh Harjani u64 __seq = 0; 37707b5b8e1SRitesh Harjani 37807b5b8e1SRitesh Harjani for_each_possible_cpu(__cpu) 37907b5b8e1SRitesh Harjani __seq += per_cpu(discard_pa_seq, __cpu); 38007b5b8e1SRitesh Harjani return __seq; 38107b5b8e1SRitesh Harjani } 38207b5b8e1SRitesh Harjani 383ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 384ffad0a44SAneesh Kumar K.V { 385c9de560dSAlex Tomas #if BITS_PER_LONG == 64 386ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 387ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 388c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 389ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 390ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 391c9de560dSAlex Tomas #else 392c9de560dSAlex Tomas #error "how many bits you are?!" 393c9de560dSAlex Tomas #endif 394ffad0a44SAneesh Kumar K.V return addr; 395ffad0a44SAneesh Kumar K.V } 396c9de560dSAlex Tomas 397c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 398c9de560dSAlex Tomas { 399c9de560dSAlex Tomas /* 400c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 401c9de560dSAlex Tomas * needs unsigned long aligned address 402c9de560dSAlex Tomas */ 403ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 404c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 405c9de560dSAlex Tomas } 406c9de560dSAlex Tomas 407c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 408c9de560dSAlex Tomas { 409ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 410c9de560dSAlex Tomas ext4_set_bit(bit, addr); 411c9de560dSAlex Tomas } 412c9de560dSAlex Tomas 413c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 414c9de560dSAlex Tomas { 415ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 416c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 417c9de560dSAlex Tomas } 418c9de560dSAlex Tomas 419eabe0444SAndrey Sidorov static inline int mb_test_and_clear_bit(int bit, void *addr) 420eabe0444SAndrey Sidorov { 421eabe0444SAndrey Sidorov addr = mb_correct_addr_and_bit(&bit, addr); 422eabe0444SAndrey Sidorov return ext4_test_and_clear_bit(bit, addr); 423eabe0444SAndrey Sidorov } 424eabe0444SAndrey Sidorov 425ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 426ffad0a44SAneesh Kumar K.V { 427e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 428ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 429e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 430ffad0a44SAneesh Kumar K.V start += fix; 431ffad0a44SAneesh Kumar K.V 432e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 433e7dfb246SAneesh Kumar K.V if (ret > max) 434e7dfb246SAneesh Kumar K.V return max; 435e7dfb246SAneesh Kumar K.V return ret; 436ffad0a44SAneesh Kumar K.V } 437ffad0a44SAneesh Kumar K.V 438ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 439ffad0a44SAneesh Kumar K.V { 440e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 441ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 442e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 443ffad0a44SAneesh Kumar K.V start += fix; 444ffad0a44SAneesh Kumar K.V 445e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 446e7dfb246SAneesh Kumar K.V if (ret > max) 447e7dfb246SAneesh Kumar K.V return max; 448e7dfb246SAneesh Kumar K.V return ret; 449ffad0a44SAneesh Kumar K.V } 450ffad0a44SAneesh Kumar K.V 451c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 452c9de560dSAlex Tomas { 453c9de560dSAlex Tomas char *bb; 454c9de560dSAlex Tomas 455c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 456c9de560dSAlex Tomas BUG_ON(max == NULL); 457c9de560dSAlex Tomas 458c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 459c9de560dSAlex Tomas *max = 0; 460c9de560dSAlex Tomas return NULL; 461c9de560dSAlex Tomas } 462c9de560dSAlex Tomas 463c9de560dSAlex Tomas /* at order 0 we see each particular block */ 46484b775a3SColy Li if (order == 0) { 465c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 466c5e8f3f3STheodore Ts'o return e4b->bd_bitmap; 46784b775a3SColy Li } 468c9de560dSAlex Tomas 469c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 470c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 471c9de560dSAlex Tomas 472c9de560dSAlex Tomas return bb; 473c9de560dSAlex Tomas } 474c9de560dSAlex Tomas 475c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 476c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 477c9de560dSAlex Tomas int first, int count) 478c9de560dSAlex Tomas { 479c9de560dSAlex Tomas int i; 480c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 481c9de560dSAlex Tomas 482c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 483c9de560dSAlex Tomas return; 484bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 485c9de560dSAlex Tomas for (i = 0; i < count; i++) { 486c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 487c9de560dSAlex Tomas ext4_fsblk_t blocknr; 4885661bd68SAkinobu Mita 4895661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 49053accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 4915d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 492e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 493e29136f8STheodore Ts'o blocknr, 494e29136f8STheodore Ts'o "freeing block already freed " 495e29136f8STheodore Ts'o "(bit %u)", 496e29136f8STheodore Ts'o first + i); 497736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 498736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 499c9de560dSAlex Tomas } 500c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 501c9de560dSAlex Tomas } 502c9de560dSAlex Tomas } 503c9de560dSAlex Tomas 504c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 505c9de560dSAlex Tomas { 506c9de560dSAlex Tomas int i; 507c9de560dSAlex Tomas 508c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 509c9de560dSAlex Tomas return; 510bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 511c9de560dSAlex Tomas for (i = 0; i < count; i++) { 512c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 513c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 514c9de560dSAlex Tomas } 515c9de560dSAlex Tomas } 516c9de560dSAlex Tomas 517c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 518c9de560dSAlex Tomas { 519eb2b8ebbSRitesh Harjani if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 520eb2b8ebbSRitesh Harjani return; 521c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 522c9de560dSAlex Tomas unsigned char *b1, *b2; 523c9de560dSAlex Tomas int i; 524c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 525c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 526c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 527c9de560dSAlex Tomas if (b1[i] != b2[i]) { 5289d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 5299d8b9ec4STheodore Ts'o "corruption in group %u " 5304776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 5319d8b9ec4STheodore Ts'o "on disk/prealloc", 532c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 533c9de560dSAlex Tomas BUG(); 534c9de560dSAlex Tomas } 535c9de560dSAlex Tomas } 536c9de560dSAlex Tomas } 537c9de560dSAlex Tomas } 538c9de560dSAlex Tomas 539a3450215SRitesh Harjani static void mb_group_bb_bitmap_alloc(struct super_block *sb, 540a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 541a3450215SRitesh Harjani { 542a3450215SRitesh Harjani struct buffer_head *bh; 543a3450215SRitesh Harjani 544a3450215SRitesh Harjani grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); 545eb2b8ebbSRitesh Harjani if (!grp->bb_bitmap) 546eb2b8ebbSRitesh Harjani return; 547a3450215SRitesh Harjani 548a3450215SRitesh Harjani bh = ext4_read_block_bitmap(sb, group); 549eb2b8ebbSRitesh Harjani if (IS_ERR_OR_NULL(bh)) { 550eb2b8ebbSRitesh Harjani kfree(grp->bb_bitmap); 551eb2b8ebbSRitesh Harjani grp->bb_bitmap = NULL; 552eb2b8ebbSRitesh Harjani return; 553eb2b8ebbSRitesh Harjani } 554a3450215SRitesh Harjani 555a3450215SRitesh Harjani memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize); 556a3450215SRitesh Harjani put_bh(bh); 557a3450215SRitesh Harjani } 558a3450215SRitesh Harjani 559a3450215SRitesh Harjani static void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 560a3450215SRitesh Harjani { 561a3450215SRitesh Harjani kfree(grp->bb_bitmap); 562a3450215SRitesh Harjani } 563a3450215SRitesh Harjani 564c9de560dSAlex Tomas #else 565c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 566c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 567c9de560dSAlex Tomas { 568c9de560dSAlex Tomas return; 569c9de560dSAlex Tomas } 570c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 571c9de560dSAlex Tomas int first, int count) 572c9de560dSAlex Tomas { 573c9de560dSAlex Tomas return; 574c9de560dSAlex Tomas } 575c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 576c9de560dSAlex Tomas { 577c9de560dSAlex Tomas return; 578c9de560dSAlex Tomas } 579a3450215SRitesh Harjani 580a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_alloc(struct super_block *sb, 581a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 582a3450215SRitesh Harjani { 583a3450215SRitesh Harjani return; 584a3450215SRitesh Harjani } 585a3450215SRitesh Harjani 586a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 587a3450215SRitesh Harjani { 588a3450215SRitesh Harjani return; 589a3450215SRitesh Harjani } 590c9de560dSAlex Tomas #endif 591c9de560dSAlex Tomas 592c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 593c9de560dSAlex Tomas 594c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 595c9de560dSAlex Tomas do { \ 596c9de560dSAlex Tomas if (!(assert)) { \ 597c9de560dSAlex Tomas printk(KERN_EMERG \ 598c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 599c9de560dSAlex Tomas function, file, line, # assert); \ 600c9de560dSAlex Tomas BUG(); \ 601c9de560dSAlex Tomas } \ 602c9de560dSAlex Tomas } while (0) 603c9de560dSAlex Tomas 604c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 605c9de560dSAlex Tomas const char *function, int line) 606c9de560dSAlex Tomas { 607c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 608c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 609c9de560dSAlex Tomas int max; 610c9de560dSAlex Tomas int max2; 611c9de560dSAlex Tomas int i; 612c9de560dSAlex Tomas int j; 613c9de560dSAlex Tomas int k; 614c9de560dSAlex Tomas int count; 615c9de560dSAlex Tomas struct ext4_group_info *grp; 616c9de560dSAlex Tomas int fragments = 0; 617c9de560dSAlex Tomas int fstart; 618c9de560dSAlex Tomas struct list_head *cur; 619c9de560dSAlex Tomas void *buddy; 620c9de560dSAlex Tomas void *buddy2; 621c9de560dSAlex Tomas 622c9de560dSAlex Tomas { 623c9de560dSAlex Tomas static int mb_check_counter; 624c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 625c9de560dSAlex Tomas return 0; 626c9de560dSAlex Tomas } 627c9de560dSAlex Tomas 628c9de560dSAlex Tomas while (order > 1) { 629c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 630c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 631c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 632c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 633c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 634c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 635c9de560dSAlex Tomas 636c9de560dSAlex Tomas count = 0; 637c9de560dSAlex Tomas for (i = 0; i < max; i++) { 638c9de560dSAlex Tomas 639c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 640c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 641c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 642c9de560dSAlex Tomas MB_CHECK_ASSERT( 643c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 644c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 645c9de560dSAlex Tomas MB_CHECK_ASSERT( 646c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 647c9de560dSAlex Tomas } 648c9de560dSAlex Tomas continue; 649c9de560dSAlex Tomas } 650c9de560dSAlex Tomas 6510a10da73SRobin Dong /* both bits in buddy2 must be 1 */ 652c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 653c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 654c9de560dSAlex Tomas 655c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 656c9de560dSAlex Tomas k = (i * (1 << order)) + j; 657c9de560dSAlex Tomas MB_CHECK_ASSERT( 658c5e8f3f3STheodore Ts'o !mb_test_bit(k, e4b->bd_bitmap)); 659c9de560dSAlex Tomas } 660c9de560dSAlex Tomas count++; 661c9de560dSAlex Tomas } 662c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 663c9de560dSAlex Tomas order--; 664c9de560dSAlex Tomas } 665c9de560dSAlex Tomas 666c9de560dSAlex Tomas fstart = -1; 667c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 668c9de560dSAlex Tomas for (i = 0; i < max; i++) { 669c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 670c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 671c9de560dSAlex Tomas if (fstart == -1) { 672c9de560dSAlex Tomas fragments++; 673c9de560dSAlex Tomas fstart = i; 674c9de560dSAlex Tomas } 675c9de560dSAlex Tomas continue; 676c9de560dSAlex Tomas } 677c9de560dSAlex Tomas fstart = -1; 678c9de560dSAlex Tomas /* check used bits only */ 679c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 680c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 681c9de560dSAlex Tomas k = i >> j; 682c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 683c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 684c9de560dSAlex Tomas } 685c9de560dSAlex Tomas } 686c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 687c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 688c9de560dSAlex Tomas 689c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 690c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 691c9de560dSAlex Tomas ext4_group_t groupnr; 692c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 69360bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 69460bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 695c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 69660bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 697c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 698c9de560dSAlex Tomas } 699c9de560dSAlex Tomas return 0; 700c9de560dSAlex Tomas } 701c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 702c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 70346e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 704c9de560dSAlex Tomas #else 705c9de560dSAlex Tomas #define mb_check_buddy(e4b) 706c9de560dSAlex Tomas #endif 707c9de560dSAlex Tomas 7087c786059SColy Li /* 7097c786059SColy Li * Divide blocks started from @first with length @len into 7107c786059SColy Li * smaller chunks with power of 2 blocks. 7117c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 7127c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 7137c786059SColy Li */ 714c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 715a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 716c9de560dSAlex Tomas struct ext4_group_info *grp) 717c9de560dSAlex Tomas { 718c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 719a36b4498SEric Sandeen ext4_grpblk_t min; 720a36b4498SEric Sandeen ext4_grpblk_t max; 721a36b4498SEric Sandeen ext4_grpblk_t chunk; 72269e43e8cSChandan Rajendra unsigned int border; 723c9de560dSAlex Tomas 7247137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 725c9de560dSAlex Tomas 726c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 727c9de560dSAlex Tomas 728c9de560dSAlex Tomas while (len > 0) { 729c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 730c9de560dSAlex Tomas max = ffs(first | border) - 1; 731c9de560dSAlex Tomas 732c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 733c9de560dSAlex Tomas min = fls(len) - 1; 734c9de560dSAlex Tomas 735c9de560dSAlex Tomas if (max < min) 736c9de560dSAlex Tomas min = max; 737c9de560dSAlex Tomas chunk = 1 << min; 738c9de560dSAlex Tomas 739c9de560dSAlex Tomas /* mark multiblock chunks only */ 740c9de560dSAlex Tomas grp->bb_counters[min]++; 741c9de560dSAlex Tomas if (min > 0) 742c9de560dSAlex Tomas mb_clear_bit(first >> min, 743c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 744c9de560dSAlex Tomas 745c9de560dSAlex Tomas len -= chunk; 746c9de560dSAlex Tomas first += chunk; 747c9de560dSAlex Tomas } 748c9de560dSAlex Tomas } 749c9de560dSAlex Tomas 7508a57d9d6SCurt Wohlgemuth /* 7518a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 7528a57d9d6SCurt Wohlgemuth * group. 7538a57d9d6SCurt Wohlgemuth */ 7548a57d9d6SCurt Wohlgemuth static void 7558a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 7568a57d9d6SCurt Wohlgemuth { 7578a57d9d6SCurt Wohlgemuth int i; 7588a57d9d6SCurt Wohlgemuth int bits; 7598a57d9d6SCurt Wohlgemuth 7608a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = -1; /* uninit */ 7618a57d9d6SCurt Wohlgemuth 7628a57d9d6SCurt Wohlgemuth bits = sb->s_blocksize_bits + 1; 7638a57d9d6SCurt Wohlgemuth for (i = bits; i >= 0; i--) { 7648a57d9d6SCurt Wohlgemuth if (grp->bb_counters[i] > 0) { 7658a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 7668a57d9d6SCurt Wohlgemuth break; 7678a57d9d6SCurt Wohlgemuth } 7688a57d9d6SCurt Wohlgemuth } 7698a57d9d6SCurt Wohlgemuth } 7708a57d9d6SCurt Wohlgemuth 771089ceeccSEric Sandeen static noinline_for_stack 772089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 773c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 774c9de560dSAlex Tomas { 775c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 776e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 7777137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 778a36b4498SEric Sandeen ext4_grpblk_t i = 0; 779a36b4498SEric Sandeen ext4_grpblk_t first; 780a36b4498SEric Sandeen ext4_grpblk_t len; 781c9de560dSAlex Tomas unsigned free = 0; 782c9de560dSAlex Tomas unsigned fragments = 0; 783c9de560dSAlex Tomas unsigned long long period = get_cycles(); 784c9de560dSAlex Tomas 785c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 786c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 787ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 788c9de560dSAlex Tomas grp->bb_first_free = i; 789c9de560dSAlex Tomas while (i < max) { 790c9de560dSAlex Tomas fragments++; 791c9de560dSAlex Tomas first = i; 792ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 793c9de560dSAlex Tomas len = i - first; 794c9de560dSAlex Tomas free += len; 795c9de560dSAlex Tomas if (len > 1) 796c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 797c9de560dSAlex Tomas else 798c9de560dSAlex Tomas grp->bb_counters[0]++; 799c9de560dSAlex Tomas if (i < max) 800ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 801c9de560dSAlex Tomas } 802c9de560dSAlex Tomas grp->bb_fragments = fragments; 803c9de560dSAlex Tomas 804c9de560dSAlex Tomas if (free != grp->bb_free) { 805e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 80694d4c066STheodore Ts'o "block bitmap and bg descriptor " 80794d4c066STheodore Ts'o "inconsistent: %u vs %u free clusters", 808e29136f8STheodore Ts'o free, grp->bb_free); 809e56eb659SAneesh Kumar K.V /* 810163a203dSDarrick J. Wong * If we intend to continue, we consider group descriptor 811e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 812e56eb659SAneesh Kumar K.V */ 813c9de560dSAlex Tomas grp->bb_free = free; 814db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, group, 815db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 816c9de560dSAlex Tomas } 8178a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 818c9de560dSAlex Tomas 819c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 820c9de560dSAlex Tomas 821c9de560dSAlex Tomas period = get_cycles() - period; 82249598e04SJun Piao spin_lock(&sbi->s_bal_lock); 82349598e04SJun Piao sbi->s_mb_buddies_generated++; 82449598e04SJun Piao sbi->s_mb_generation_time += period; 82549598e04SJun Piao spin_unlock(&sbi->s_bal_lock); 826c9de560dSAlex Tomas } 827c9de560dSAlex Tomas 828eabe0444SAndrey Sidorov static void mb_regenerate_buddy(struct ext4_buddy *e4b) 829eabe0444SAndrey Sidorov { 830eabe0444SAndrey Sidorov int count; 831eabe0444SAndrey Sidorov int order = 1; 832eabe0444SAndrey Sidorov void *buddy; 833eabe0444SAndrey Sidorov 834eabe0444SAndrey Sidorov while ((buddy = mb_find_buddy(e4b, order++, &count))) { 835eabe0444SAndrey Sidorov ext4_set_bits(buddy, 0, count); 836eabe0444SAndrey Sidorov } 837eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments = 0; 838eabe0444SAndrey Sidorov memset(e4b->bd_info->bb_counters, 0, 839eabe0444SAndrey Sidorov sizeof(*e4b->bd_info->bb_counters) * 840eabe0444SAndrey Sidorov (e4b->bd_sb->s_blocksize_bits + 2)); 841eabe0444SAndrey Sidorov 842eabe0444SAndrey Sidorov ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy, 843eabe0444SAndrey Sidorov e4b->bd_bitmap, e4b->bd_group); 844eabe0444SAndrey Sidorov } 845eabe0444SAndrey Sidorov 846c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 847c9de560dSAlex Tomas * for convenience. The information regarding each group 848c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 849c9de560dSAlex Tomas * block bitmap and buddy information. The information are 850c9de560dSAlex Tomas * stored in the inode as 851c9de560dSAlex Tomas * 852c9de560dSAlex Tomas * { page } 853c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 854c9de560dSAlex Tomas * 855c9de560dSAlex Tomas * 856c9de560dSAlex Tomas * one block each for bitmap and buddy information. 857c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 858ea1754a0SKirill A. Shutemov * contain blocks_per_page (PAGE_SIZE / blocksize) blocks. 859c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 860c9de560dSAlex Tomas * is blocks_per_page/2 8618a57d9d6SCurt Wohlgemuth * 8628a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 8638a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 864c9de560dSAlex Tomas */ 865c9de560dSAlex Tomas 866adb7ef60SKonstantin Khlebnikov static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) 867c9de560dSAlex Tomas { 8688df9675fSTheodore Ts'o ext4_group_t ngroups; 869c9de560dSAlex Tomas int blocksize; 870c9de560dSAlex Tomas int blocks_per_page; 871c9de560dSAlex Tomas int groups_per_page; 872c9de560dSAlex Tomas int err = 0; 873c9de560dSAlex Tomas int i; 874813e5727STheodore Ts'o ext4_group_t first_group, group; 875c9de560dSAlex Tomas int first_block; 876c9de560dSAlex Tomas struct super_block *sb; 877c9de560dSAlex Tomas struct buffer_head *bhs; 878fa77dcfaSDarrick J. Wong struct buffer_head **bh = NULL; 879c9de560dSAlex Tomas struct inode *inode; 880c9de560dSAlex Tomas char *data; 881c9de560dSAlex Tomas char *bitmap; 8829b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 883c9de560dSAlex Tomas 884c9de560dSAlex Tomas inode = page->mapping->host; 885c9de560dSAlex Tomas sb = inode->i_sb; 8868df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 88793407472SFabian Frederick blocksize = i_blocksize(inode); 88809cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / blocksize; 889c9de560dSAlex Tomas 890d3df1453SRitesh Harjani mb_debug(sb, "init page %lu\n", page->index); 891d3df1453SRitesh Harjani 892c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 893c9de560dSAlex Tomas if (groups_per_page == 0) 894c9de560dSAlex Tomas groups_per_page = 1; 895c9de560dSAlex Tomas 896c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 897c9de560dSAlex Tomas if (groups_per_page > 1) { 898c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 899adb7ef60SKonstantin Khlebnikov bh = kzalloc(i, gfp); 900813e5727STheodore Ts'o if (bh == NULL) { 901813e5727STheodore Ts'o err = -ENOMEM; 902c9de560dSAlex Tomas goto out; 903813e5727STheodore Ts'o } 904c9de560dSAlex Tomas } else 905c9de560dSAlex Tomas bh = &bhs; 906c9de560dSAlex Tomas 907c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 908c9de560dSAlex Tomas 909c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 910813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 911813e5727STheodore Ts'o if (group >= ngroups) 912c9de560dSAlex Tomas break; 913c9de560dSAlex Tomas 914813e5727STheodore Ts'o grinfo = ext4_get_group_info(sb, group); 9159b8b7d35SAmir Goldstein /* 9169b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 9179b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 9189b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 9199b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 9209b8b7d35SAmir Goldstein */ 9219b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 9229b8b7d35SAmir Goldstein bh[i] = NULL; 9239b8b7d35SAmir Goldstein continue; 9249b8b7d35SAmir Goldstein } 925cfd73237SAlex Zhuravlev bh[i] = ext4_read_block_bitmap_nowait(sb, group, false); 9269008a58eSDarrick J. Wong if (IS_ERR(bh[i])) { 9279008a58eSDarrick J. Wong err = PTR_ERR(bh[i]); 9289008a58eSDarrick J. Wong bh[i] = NULL; 929c9de560dSAlex Tomas goto out; 9302ccb5fb9SAneesh Kumar K.V } 931d3df1453SRitesh Harjani mb_debug(sb, "read bitmap for group %u\n", group); 932c9de560dSAlex Tomas } 933c9de560dSAlex Tomas 934c9de560dSAlex Tomas /* wait for I/O completion */ 935813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 9369008a58eSDarrick J. Wong int err2; 9379008a58eSDarrick J. Wong 9389008a58eSDarrick J. Wong if (!bh[i]) 9399008a58eSDarrick J. Wong continue; 9409008a58eSDarrick J. Wong err2 = ext4_wait_block_bitmap(sb, group, bh[i]); 9419008a58eSDarrick J. Wong if (!err) 9429008a58eSDarrick J. Wong err = err2; 943813e5727STheodore Ts'o } 944c9de560dSAlex Tomas 945c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 946c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 947c9de560dSAlex Tomas group = (first_block + i) >> 1; 9488df9675fSTheodore Ts'o if (group >= ngroups) 949c9de560dSAlex Tomas break; 950c9de560dSAlex Tomas 9519b8b7d35SAmir Goldstein if (!bh[group - first_group]) 9529b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 9539b8b7d35SAmir Goldstein continue; 9549b8b7d35SAmir Goldstein 955bbdc322fSLukas Czerner if (!buffer_verified(bh[group - first_group])) 956bbdc322fSLukas Czerner /* Skip faulty bitmaps */ 957bbdc322fSLukas Czerner continue; 958bbdc322fSLukas Czerner err = 0; 959bbdc322fSLukas Czerner 960c9de560dSAlex Tomas /* 961c9de560dSAlex Tomas * data carry information regarding this 962c9de560dSAlex Tomas * particular group in the format specified 963c9de560dSAlex Tomas * above 964c9de560dSAlex Tomas * 965c9de560dSAlex Tomas */ 966c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 967c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 968c9de560dSAlex Tomas 969c9de560dSAlex Tomas /* 970c9de560dSAlex Tomas * We place the buddy block and bitmap block 971c9de560dSAlex Tomas * close together 972c9de560dSAlex Tomas */ 973c9de560dSAlex Tomas if ((first_block + i) & 1) { 974c9de560dSAlex Tomas /* this is block of buddy */ 975c9de560dSAlex Tomas BUG_ON(incore == NULL); 976d3df1453SRitesh Harjani mb_debug(sb, "put buddy for group %u in page %lu/%x\n", 977c9de560dSAlex Tomas group, page->index, i * blocksize); 978f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 979c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 980c9de560dSAlex Tomas grinfo->bb_fragments = 0; 981c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 9821927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 9831927805eSEric Sandeen (sb->s_blocksize_bits+2)); 984c9de560dSAlex Tomas /* 985c9de560dSAlex Tomas * incore got set to the group block bitmap below 986c9de560dSAlex Tomas */ 9877a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 9889b8b7d35SAmir Goldstein /* init the buddy */ 9899b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 990c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 9917a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 992c9de560dSAlex Tomas incore = NULL; 993c9de560dSAlex Tomas } else { 994c9de560dSAlex Tomas /* this is block of bitmap */ 995c9de560dSAlex Tomas BUG_ON(incore != NULL); 996d3df1453SRitesh Harjani mb_debug(sb, "put bitmap for group %u in page %lu/%x\n", 997c9de560dSAlex Tomas group, page->index, i * blocksize); 998f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 999c9de560dSAlex Tomas 1000c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 1001c9de560dSAlex Tomas ext4_lock_group(sb, group); 1002c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 1003c9de560dSAlex Tomas 1004c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 1005c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 10067a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 1007c9de560dSAlex Tomas ext4_unlock_group(sb, group); 1008c9de560dSAlex Tomas 1009c9de560dSAlex Tomas /* set incore so that the buddy information can be 1010c9de560dSAlex Tomas * generated using this 1011c9de560dSAlex Tomas */ 1012c9de560dSAlex Tomas incore = data; 1013c9de560dSAlex Tomas } 1014c9de560dSAlex Tomas } 1015c9de560dSAlex Tomas SetPageUptodate(page); 1016c9de560dSAlex Tomas 1017c9de560dSAlex Tomas out: 1018c9de560dSAlex Tomas if (bh) { 10199b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 1020c9de560dSAlex Tomas brelse(bh[i]); 1021c9de560dSAlex Tomas if (bh != &bhs) 1022c9de560dSAlex Tomas kfree(bh); 1023c9de560dSAlex Tomas } 1024c9de560dSAlex Tomas return err; 1025c9de560dSAlex Tomas } 1026c9de560dSAlex Tomas 10278a57d9d6SCurt Wohlgemuth /* 10282de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 10292de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 10302de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 10312de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 1032eee4adc7SEric Sandeen */ 10332de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 1034adb7ef60SKonstantin Khlebnikov ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) 1035eee4adc7SEric Sandeen { 10362de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 10372de8807bSAmir Goldstein int block, pnum, poff; 1038eee4adc7SEric Sandeen int blocks_per_page; 10392de8807bSAmir Goldstein struct page *page; 10402de8807bSAmir Goldstein 10412de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 10422de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 1043eee4adc7SEric Sandeen 104409cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1045eee4adc7SEric Sandeen /* 1046eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 1047eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 1048eee4adc7SEric Sandeen * So for each group we need two blocks. 1049eee4adc7SEric Sandeen */ 1050eee4adc7SEric Sandeen block = group * 2; 1051eee4adc7SEric Sandeen pnum = block / blocks_per_page; 10522de8807bSAmir Goldstein poff = block % blocks_per_page; 1053adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 10542de8807bSAmir Goldstein if (!page) 1055c57ab39bSYounger Liu return -ENOMEM; 10562de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10572de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 10582de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1059eee4adc7SEric Sandeen 10602de8807bSAmir Goldstein if (blocks_per_page >= 2) { 10612de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 10622de8807bSAmir Goldstein return 0; 1063eee4adc7SEric Sandeen } 1064eee4adc7SEric Sandeen 10652de8807bSAmir Goldstein block++; 1066eee4adc7SEric Sandeen pnum = block / blocks_per_page; 1067adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 10682de8807bSAmir Goldstein if (!page) 1069c57ab39bSYounger Liu return -ENOMEM; 10702de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10712de8807bSAmir Goldstein e4b->bd_buddy_page = page; 10722de8807bSAmir Goldstein return 0; 1073eee4adc7SEric Sandeen } 1074eee4adc7SEric Sandeen 10752de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 10762de8807bSAmir Goldstein { 10772de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 10782de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 107909cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 10802de8807bSAmir Goldstein } 10812de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 10822de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 108309cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 10842de8807bSAmir Goldstein } 1085eee4adc7SEric Sandeen } 1086eee4adc7SEric Sandeen 1087eee4adc7SEric Sandeen /* 10888a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10898a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10908a57d9d6SCurt Wohlgemuth * calling this routine! 10918a57d9d6SCurt Wohlgemuth */ 1092b6a758ecSAneesh Kumar K.V static noinline_for_stack 1093adb7ef60SKonstantin Khlebnikov int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) 1094b6a758ecSAneesh Kumar K.V { 1095b6a758ecSAneesh Kumar K.V 1096b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 10972de8807bSAmir Goldstein struct ext4_buddy e4b; 10982de8807bSAmir Goldstein struct page *page; 10992de8807bSAmir Goldstein int ret = 0; 1100b6a758ecSAneesh Kumar K.V 1101b10a44c3STheodore Ts'o might_sleep(); 1102d3df1453SRitesh Harjani mb_debug(sb, "init group %u\n", group); 1103b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 1104b6a758ecSAneesh Kumar K.V /* 110508c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 110608c3a813SAneesh Kumar K.V * page which map to the group from which we are already 110708c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 110808c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 11092de8807bSAmir Goldstein * would have pinned buddy page to page cache. 11102457aec6SMel Gorman * The call to ext4_mb_get_buddy_page_lock will mark the 11112457aec6SMel Gorman * page accessed. 1112b6a758ecSAneesh Kumar K.V */ 1113adb7ef60SKonstantin Khlebnikov ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp); 11142de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1115b6a758ecSAneesh Kumar K.V /* 1116b6a758ecSAneesh Kumar K.V * somebody initialized the group 1117b6a758ecSAneesh Kumar K.V * return without doing anything 1118b6a758ecSAneesh Kumar K.V */ 1119b6a758ecSAneesh Kumar K.V goto err; 1120b6a758ecSAneesh Kumar K.V } 11212de8807bSAmir Goldstein 11222de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1123adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 11242de8807bSAmir Goldstein if (ret) 1125b6a758ecSAneesh Kumar K.V goto err; 11262de8807bSAmir Goldstein if (!PageUptodate(page)) { 1127b6a758ecSAneesh Kumar K.V ret = -EIO; 1128b6a758ecSAneesh Kumar K.V goto err; 1129b6a758ecSAneesh Kumar K.V } 1130b6a758ecSAneesh Kumar K.V 11312de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1132b6a758ecSAneesh Kumar K.V /* 1133b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1134b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1135b6a758ecSAneesh Kumar K.V * init the buddy 1136b6a758ecSAneesh Kumar K.V */ 11372de8807bSAmir Goldstein ret = 0; 1138b6a758ecSAneesh Kumar K.V goto err; 1139b6a758ecSAneesh Kumar K.V } 11402de8807bSAmir Goldstein /* init buddy cache */ 11412de8807bSAmir Goldstein page = e4b.bd_buddy_page; 1142adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp); 11432de8807bSAmir Goldstein if (ret) 11442de8807bSAmir Goldstein goto err; 11452de8807bSAmir Goldstein if (!PageUptodate(page)) { 1146b6a758ecSAneesh Kumar K.V ret = -EIO; 1147b6a758ecSAneesh Kumar K.V goto err; 1148b6a758ecSAneesh Kumar K.V } 1149b6a758ecSAneesh Kumar K.V err: 11502de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1151b6a758ecSAneesh Kumar K.V return ret; 1152b6a758ecSAneesh Kumar K.V } 1153b6a758ecSAneesh Kumar K.V 11548a57d9d6SCurt Wohlgemuth /* 11558a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 11568a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 11578a57d9d6SCurt Wohlgemuth * calling this routine! 11588a57d9d6SCurt Wohlgemuth */ 11594ddfef7bSEric Sandeen static noinline_for_stack int 1160adb7ef60SKonstantin Khlebnikov ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, 1161adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b, gfp_t gfp) 1162c9de560dSAlex Tomas { 1163c9de560dSAlex Tomas int blocks_per_page; 1164c9de560dSAlex Tomas int block; 1165c9de560dSAlex Tomas int pnum; 1166c9de560dSAlex Tomas int poff; 1167c9de560dSAlex Tomas struct page *page; 1168fdf6c7a7SShen Feng int ret; 1169920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1170920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1171920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1172c9de560dSAlex Tomas 1173b10a44c3STheodore Ts'o might_sleep(); 1174d3df1453SRitesh Harjani mb_debug(sb, "load group %u\n", group); 1175c9de560dSAlex Tomas 117609cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1177920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1178c9de560dSAlex Tomas 1179c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1180529da704STao Ma e4b->bd_info = grp; 1181c9de560dSAlex Tomas e4b->bd_sb = sb; 1182c9de560dSAlex Tomas e4b->bd_group = group; 1183c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1184c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1185c9de560dSAlex Tomas 1186f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1187f41c0750SAneesh Kumar K.V /* 1188f41c0750SAneesh Kumar K.V * we need full data about the group 1189f41c0750SAneesh Kumar K.V * to make a good selection 1190f41c0750SAneesh Kumar K.V */ 1191adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, gfp); 1192f41c0750SAneesh Kumar K.V if (ret) 1193f41c0750SAneesh Kumar K.V return ret; 1194f41c0750SAneesh Kumar K.V } 1195f41c0750SAneesh Kumar K.V 1196c9de560dSAlex Tomas /* 1197c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1198c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1199c9de560dSAlex Tomas * So for each group we need two blocks. 1200c9de560dSAlex Tomas */ 1201c9de560dSAlex Tomas block = group * 2; 1202c9de560dSAlex Tomas pnum = block / blocks_per_page; 1203c9de560dSAlex Tomas poff = block % blocks_per_page; 1204c9de560dSAlex Tomas 1205c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1206c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 12072457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1208c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1209c9de560dSAlex Tomas if (page) 1210920313a7SAneesh Kumar K.V /* 1211920313a7SAneesh Kumar K.V * drop the page reference and try 1212920313a7SAneesh Kumar K.V * to get the page with lock. If we 1213920313a7SAneesh Kumar K.V * are not uptodate that implies 1214920313a7SAneesh Kumar K.V * somebody just created the page but 1215920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1216920313a7SAneesh Kumar K.V * wait for it to initialize. 1217920313a7SAneesh Kumar K.V */ 121809cbfeafSKirill A. Shutemov put_page(page); 1219adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1220c9de560dSAlex Tomas if (page) { 1221c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1222c9de560dSAlex Tomas if (!PageUptodate(page)) { 1223adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 1224fdf6c7a7SShen Feng if (ret) { 1225fdf6c7a7SShen Feng unlock_page(page); 1226fdf6c7a7SShen Feng goto err; 1227fdf6c7a7SShen Feng } 1228c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1229c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1230c9de560dSAlex Tomas } 1231c9de560dSAlex Tomas unlock_page(page); 1232c9de560dSAlex Tomas } 1233c9de560dSAlex Tomas } 1234c57ab39bSYounger Liu if (page == NULL) { 1235c57ab39bSYounger Liu ret = -ENOMEM; 1236c57ab39bSYounger Liu goto err; 1237c57ab39bSYounger Liu } 1238c57ab39bSYounger Liu if (!PageUptodate(page)) { 1239fdf6c7a7SShen Feng ret = -EIO; 1240c9de560dSAlex Tomas goto err; 1241fdf6c7a7SShen Feng } 12422457aec6SMel Gorman 12432457aec6SMel Gorman /* Pages marked accessed already */ 1244c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1245c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1246c9de560dSAlex Tomas 1247c9de560dSAlex Tomas block++; 1248c9de560dSAlex Tomas pnum = block / blocks_per_page; 1249c9de560dSAlex Tomas poff = block % blocks_per_page; 1250c9de560dSAlex Tomas 12512457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1252c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1253c9de560dSAlex Tomas if (page) 125409cbfeafSKirill A. Shutemov put_page(page); 1255adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1256c9de560dSAlex Tomas if (page) { 1257c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1258fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1259adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b->bd_bitmap, 1260adb7ef60SKonstantin Khlebnikov gfp); 1261fdf6c7a7SShen Feng if (ret) { 1262fdf6c7a7SShen Feng unlock_page(page); 1263fdf6c7a7SShen Feng goto err; 1264fdf6c7a7SShen Feng } 1265fdf6c7a7SShen Feng } 1266c9de560dSAlex Tomas unlock_page(page); 1267c9de560dSAlex Tomas } 1268c9de560dSAlex Tomas } 1269c57ab39bSYounger Liu if (page == NULL) { 1270c57ab39bSYounger Liu ret = -ENOMEM; 1271c57ab39bSYounger Liu goto err; 1272c57ab39bSYounger Liu } 1273c57ab39bSYounger Liu if (!PageUptodate(page)) { 1274fdf6c7a7SShen Feng ret = -EIO; 1275c9de560dSAlex Tomas goto err; 1276fdf6c7a7SShen Feng } 12772457aec6SMel Gorman 12782457aec6SMel Gorman /* Pages marked accessed already */ 1279c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1280c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1281c9de560dSAlex Tomas 1282c9de560dSAlex Tomas return 0; 1283c9de560dSAlex Tomas 1284c9de560dSAlex Tomas err: 128526626f11SYang Ruirui if (page) 128609cbfeafSKirill A. Shutemov put_page(page); 1287c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 128809cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1289c9de560dSAlex Tomas if (e4b->bd_buddy_page) 129009cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1291c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1292c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1293fdf6c7a7SShen Feng return ret; 1294c9de560dSAlex Tomas } 1295c9de560dSAlex Tomas 1296adb7ef60SKonstantin Khlebnikov static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1297adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b) 1298adb7ef60SKonstantin Khlebnikov { 1299adb7ef60SKonstantin Khlebnikov return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS); 1300adb7ef60SKonstantin Khlebnikov } 1301adb7ef60SKonstantin Khlebnikov 1302e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1303c9de560dSAlex Tomas { 1304c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 130509cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1306c9de560dSAlex Tomas if (e4b->bd_buddy_page) 130709cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1308c9de560dSAlex Tomas } 1309c9de560dSAlex Tomas 1310c9de560dSAlex Tomas 1311c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1312c9de560dSAlex Tomas { 1313c9de560dSAlex Tomas int order = 1; 1314b5cb316cSNicolai Stange int bb_incr = 1 << (e4b->bd_blkbits - 1); 1315c9de560dSAlex Tomas void *bb; 1316c9de560dSAlex Tomas 1317c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 1318c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1319c9de560dSAlex Tomas 1320c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy; 1321c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1322c9de560dSAlex Tomas block = block >> 1; 1323c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1324c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1325c9de560dSAlex Tomas return order; 1326c9de560dSAlex Tomas } 1327b5cb316cSNicolai Stange bb += bb_incr; 1328b5cb316cSNicolai Stange bb_incr >>= 1; 1329c9de560dSAlex Tomas order++; 1330c9de560dSAlex Tomas } 1331c9de560dSAlex Tomas return 0; 1332c9de560dSAlex Tomas } 1333c9de560dSAlex Tomas 1334955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1335c9de560dSAlex Tomas { 1336c9de560dSAlex Tomas __u32 *addr; 1337c9de560dSAlex Tomas 1338c9de560dSAlex Tomas len = cur + len; 1339c9de560dSAlex Tomas while (cur < len) { 1340c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1341c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1342c9de560dSAlex Tomas addr = bm + (cur >> 3); 1343c9de560dSAlex Tomas *addr = 0; 1344c9de560dSAlex Tomas cur += 32; 1345c9de560dSAlex Tomas continue; 1346c9de560dSAlex Tomas } 1347e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1348c9de560dSAlex Tomas cur++; 1349c9de560dSAlex Tomas } 1350c9de560dSAlex Tomas } 1351c9de560dSAlex Tomas 1352eabe0444SAndrey Sidorov /* clear bits in given range 1353eabe0444SAndrey Sidorov * will return first found zero bit if any, -1 otherwise 1354eabe0444SAndrey Sidorov */ 1355eabe0444SAndrey Sidorov static int mb_test_and_clear_bits(void *bm, int cur, int len) 1356eabe0444SAndrey Sidorov { 1357eabe0444SAndrey Sidorov __u32 *addr; 1358eabe0444SAndrey Sidorov int zero_bit = -1; 1359eabe0444SAndrey Sidorov 1360eabe0444SAndrey Sidorov len = cur + len; 1361eabe0444SAndrey Sidorov while (cur < len) { 1362eabe0444SAndrey Sidorov if ((cur & 31) == 0 && (len - cur) >= 32) { 1363eabe0444SAndrey Sidorov /* fast path: clear whole word at once */ 1364eabe0444SAndrey Sidorov addr = bm + (cur >> 3); 1365eabe0444SAndrey Sidorov if (*addr != (__u32)(-1) && zero_bit == -1) 1366eabe0444SAndrey Sidorov zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); 1367eabe0444SAndrey Sidorov *addr = 0; 1368eabe0444SAndrey Sidorov cur += 32; 1369eabe0444SAndrey Sidorov continue; 1370eabe0444SAndrey Sidorov } 1371eabe0444SAndrey Sidorov if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) 1372eabe0444SAndrey Sidorov zero_bit = cur; 1373eabe0444SAndrey Sidorov cur++; 1374eabe0444SAndrey Sidorov } 1375eabe0444SAndrey Sidorov 1376eabe0444SAndrey Sidorov return zero_bit; 1377eabe0444SAndrey Sidorov } 1378eabe0444SAndrey Sidorov 1379c3e94d1dSYongqiang Yang void ext4_set_bits(void *bm, int cur, int len) 1380c9de560dSAlex Tomas { 1381c9de560dSAlex Tomas __u32 *addr; 1382c9de560dSAlex Tomas 1383c9de560dSAlex Tomas len = cur + len; 1384c9de560dSAlex Tomas while (cur < len) { 1385c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1386c9de560dSAlex Tomas /* fast path: set whole word at once */ 1387c9de560dSAlex Tomas addr = bm + (cur >> 3); 1388c9de560dSAlex Tomas *addr = 0xffffffff; 1389c9de560dSAlex Tomas cur += 32; 1390c9de560dSAlex Tomas continue; 1391c9de560dSAlex Tomas } 1392e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1393c9de560dSAlex Tomas cur++; 1394c9de560dSAlex Tomas } 1395c9de560dSAlex Tomas } 1396c9de560dSAlex Tomas 1397eabe0444SAndrey Sidorov /* 1398eabe0444SAndrey Sidorov * _________________________________________________________________ */ 1399eabe0444SAndrey Sidorov 1400eabe0444SAndrey Sidorov static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) 1401eabe0444SAndrey Sidorov { 1402eabe0444SAndrey Sidorov if (mb_test_bit(*bit + side, bitmap)) { 1403eabe0444SAndrey Sidorov mb_clear_bit(*bit, bitmap); 1404eabe0444SAndrey Sidorov (*bit) -= side; 1405eabe0444SAndrey Sidorov return 1; 1406eabe0444SAndrey Sidorov } 1407eabe0444SAndrey Sidorov else { 1408eabe0444SAndrey Sidorov (*bit) += side; 1409eabe0444SAndrey Sidorov mb_set_bit(*bit, bitmap); 1410eabe0444SAndrey Sidorov return -1; 1411eabe0444SAndrey Sidorov } 1412eabe0444SAndrey Sidorov } 1413eabe0444SAndrey Sidorov 1414eabe0444SAndrey Sidorov static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) 1415eabe0444SAndrey Sidorov { 1416eabe0444SAndrey Sidorov int max; 1417eabe0444SAndrey Sidorov int order = 1; 1418eabe0444SAndrey Sidorov void *buddy = mb_find_buddy(e4b, order, &max); 1419eabe0444SAndrey Sidorov 1420eabe0444SAndrey Sidorov while (buddy) { 1421eabe0444SAndrey Sidorov void *buddy2; 1422eabe0444SAndrey Sidorov 1423eabe0444SAndrey Sidorov /* Bits in range [first; last] are known to be set since 1424eabe0444SAndrey Sidorov * corresponding blocks were allocated. Bits in range 1425eabe0444SAndrey Sidorov * (first; last) will stay set because they form buddies on 1426eabe0444SAndrey Sidorov * upper layer. We just deal with borders if they don't 1427eabe0444SAndrey Sidorov * align with upper layer and then go up. 1428eabe0444SAndrey Sidorov * Releasing entire group is all about clearing 1429eabe0444SAndrey Sidorov * single bit of highest order buddy. 1430eabe0444SAndrey Sidorov */ 1431eabe0444SAndrey Sidorov 1432eabe0444SAndrey Sidorov /* Example: 1433eabe0444SAndrey Sidorov * --------------------------------- 1434eabe0444SAndrey Sidorov * | 1 | 1 | 1 | 1 | 1435eabe0444SAndrey Sidorov * --------------------------------- 1436eabe0444SAndrey Sidorov * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1437eabe0444SAndrey Sidorov * --------------------------------- 1438eabe0444SAndrey Sidorov * 0 1 2 3 4 5 6 7 1439eabe0444SAndrey Sidorov * \_____________________/ 1440eabe0444SAndrey Sidorov * 1441eabe0444SAndrey Sidorov * Neither [1] nor [6] is aligned to above layer. 1442eabe0444SAndrey Sidorov * Left neighbour [0] is free, so mark it busy, 1443eabe0444SAndrey Sidorov * decrease bb_counters and extend range to 1444eabe0444SAndrey Sidorov * [0; 6] 1445eabe0444SAndrey Sidorov * Right neighbour [7] is busy. It can't be coaleasced with [6], so 1446eabe0444SAndrey Sidorov * mark [6] free, increase bb_counters and shrink range to 1447eabe0444SAndrey Sidorov * [0; 5]. 1448eabe0444SAndrey Sidorov * Then shift range to [0; 2], go up and do the same. 1449eabe0444SAndrey Sidorov */ 1450eabe0444SAndrey Sidorov 1451eabe0444SAndrey Sidorov 1452eabe0444SAndrey Sidorov if (first & 1) 1453eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); 1454eabe0444SAndrey Sidorov if (!(last & 1)) 1455eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); 1456eabe0444SAndrey Sidorov if (first > last) 1457eabe0444SAndrey Sidorov break; 1458eabe0444SAndrey Sidorov order++; 1459eabe0444SAndrey Sidorov 1460eabe0444SAndrey Sidorov if (first == last || !(buddy2 = mb_find_buddy(e4b, order, &max))) { 1461eabe0444SAndrey Sidorov mb_clear_bits(buddy, first, last - first + 1); 1462eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order - 1] += last - first + 1; 1463eabe0444SAndrey Sidorov break; 1464eabe0444SAndrey Sidorov } 1465eabe0444SAndrey Sidorov first >>= 1; 1466eabe0444SAndrey Sidorov last >>= 1; 1467eabe0444SAndrey Sidorov buddy = buddy2; 1468eabe0444SAndrey Sidorov } 1469eabe0444SAndrey Sidorov } 1470eabe0444SAndrey Sidorov 14717e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1472c9de560dSAlex Tomas int first, int count) 1473c9de560dSAlex Tomas { 1474eabe0444SAndrey Sidorov int left_is_free = 0; 1475eabe0444SAndrey Sidorov int right_is_free = 0; 1476eabe0444SAndrey Sidorov int block; 1477eabe0444SAndrey Sidorov int last = first + count - 1; 1478c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1479c9de560dSAlex Tomas 1480c99d1e6eSTheodore Ts'o if (WARN_ON(count == 0)) 1481c99d1e6eSTheodore Ts'o return; 1482eabe0444SAndrey Sidorov BUG_ON(last >= (sb->s_blocksize << 3)); 1483bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1484163a203dSDarrick J. Wong /* Don't bother if the block group is corrupt. */ 1485163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) 1486163a203dSDarrick J. Wong return; 1487163a203dSDarrick J. Wong 1488c9de560dSAlex Tomas mb_check_buddy(e4b); 1489c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1490c9de560dSAlex Tomas 149107b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 1492c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1493c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1494c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1495c9de560dSAlex Tomas 1496eabe0444SAndrey Sidorov /* access memory sequentially: check left neighbour, 1497eabe0444SAndrey Sidorov * clear range and then check right neighbour 1498eabe0444SAndrey Sidorov */ 1499c9de560dSAlex Tomas if (first != 0) 1500eabe0444SAndrey Sidorov left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); 1501eabe0444SAndrey Sidorov block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); 1502eabe0444SAndrey Sidorov if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) 1503eabe0444SAndrey Sidorov right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); 1504c9de560dSAlex Tomas 1505eabe0444SAndrey Sidorov if (unlikely(block != -1)) { 1506e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 1507c9de560dSAlex Tomas ext4_fsblk_t blocknr; 15085661bd68SAkinobu Mita 15095661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 151049598e04SJun Piao blocknr += EXT4_C2B(sbi, block); 15115d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1512e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1513e29136f8STheodore Ts'o blocknr, 1514e29136f8STheodore Ts'o "freeing already freed block " 1515163a203dSDarrick J. Wong "(bit %u); block bitmap corrupt.", 1516163a203dSDarrick J. Wong block); 1517db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 1518db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1519eabe0444SAndrey Sidorov mb_regenerate_buddy(e4b); 1520eabe0444SAndrey Sidorov goto done; 1521c9de560dSAlex Tomas } 1522c9de560dSAlex Tomas 1523eabe0444SAndrey Sidorov /* let's maintain fragments counter */ 1524eabe0444SAndrey Sidorov if (left_is_free && right_is_free) 1525eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments--; 1526eabe0444SAndrey Sidorov else if (!left_is_free && !right_is_free) 1527eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments++; 1528c9de560dSAlex Tomas 1529eabe0444SAndrey Sidorov /* buddy[0] == bd_bitmap is a special case, so handle 1530eabe0444SAndrey Sidorov * it right away and let mb_buddy_mark_free stay free of 1531eabe0444SAndrey Sidorov * zero order checks. 1532eabe0444SAndrey Sidorov * Check if neighbours are to be coaleasced, 1533eabe0444SAndrey Sidorov * adjust bitmap bb_counters and borders appropriately. 1534eabe0444SAndrey Sidorov */ 1535eabe0444SAndrey Sidorov if (first & 1) { 1536eabe0444SAndrey Sidorov first += !left_is_free; 1537eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; 1538c9de560dSAlex Tomas } 1539eabe0444SAndrey Sidorov if (!(last & 1)) { 1540eabe0444SAndrey Sidorov last -= !right_is_free; 1541eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; 1542c9de560dSAlex Tomas } 1543eabe0444SAndrey Sidorov 1544eabe0444SAndrey Sidorov if (first <= last) 1545eabe0444SAndrey Sidorov mb_buddy_mark_free(e4b, first >> 1, last >> 1); 1546eabe0444SAndrey Sidorov 1547eabe0444SAndrey Sidorov done: 15488a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1549c9de560dSAlex Tomas mb_check_buddy(e4b); 1550c9de560dSAlex Tomas } 1551c9de560dSAlex Tomas 155215c006a2SRobin Dong static int mb_find_extent(struct ext4_buddy *e4b, int block, 1553c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1554c9de560dSAlex Tomas { 1555c9de560dSAlex Tomas int next = block; 155615c006a2SRobin Dong int max, order; 1557c9de560dSAlex Tomas void *buddy; 1558c9de560dSAlex Tomas 1559bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1560c9de560dSAlex Tomas BUG_ON(ex == NULL); 1561c9de560dSAlex Tomas 156215c006a2SRobin Dong buddy = mb_find_buddy(e4b, 0, &max); 1563c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1564c9de560dSAlex Tomas BUG_ON(block >= max); 1565c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1566c9de560dSAlex Tomas ex->fe_len = 0; 1567c9de560dSAlex Tomas ex->fe_start = 0; 1568c9de560dSAlex Tomas ex->fe_group = 0; 1569c9de560dSAlex Tomas return 0; 1570c9de560dSAlex Tomas } 1571c9de560dSAlex Tomas 1572c9de560dSAlex Tomas /* find actual order */ 1573c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1574c9de560dSAlex Tomas block = block >> order; 1575c9de560dSAlex Tomas 1576c9de560dSAlex Tomas ex->fe_len = 1 << order; 1577c9de560dSAlex Tomas ex->fe_start = block << order; 1578c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1579c9de560dSAlex Tomas 1580c9de560dSAlex Tomas /* calc difference from given start */ 1581c9de560dSAlex Tomas next = next - ex->fe_start; 1582c9de560dSAlex Tomas ex->fe_len -= next; 1583c9de560dSAlex Tomas ex->fe_start += next; 1584c9de560dSAlex Tomas 1585c9de560dSAlex Tomas while (needed > ex->fe_len && 1586d8ec0c39SAlan Cox mb_find_buddy(e4b, order, &max)) { 1587c9de560dSAlex Tomas 1588c9de560dSAlex Tomas if (block + 1 >= max) 1589c9de560dSAlex Tomas break; 1590c9de560dSAlex Tomas 1591c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1592c5e8f3f3STheodore Ts'o if (mb_test_bit(next, e4b->bd_bitmap)) 1593c9de560dSAlex Tomas break; 1594c9de560dSAlex Tomas 1595b051d8dcSRobin Dong order = mb_find_order_for_block(e4b, next); 1596c9de560dSAlex Tomas 1597c9de560dSAlex Tomas block = next >> order; 1598c9de560dSAlex Tomas ex->fe_len += 1 << order; 1599c9de560dSAlex Tomas } 1600c9de560dSAlex Tomas 160131562b95SJan Kara if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { 160243c73221STheodore Ts'o /* Should never happen! (but apparently sometimes does?!?) */ 160343c73221STheodore Ts'o WARN_ON(1); 160443c73221STheodore Ts'o ext4_error(e4b->bd_sb, "corruption or bug in mb_find_extent " 160543c73221STheodore Ts'o "block=%d, order=%d needed=%d ex=%u/%d/%d@%u", 160643c73221STheodore Ts'o block, order, needed, ex->fe_group, ex->fe_start, 160743c73221STheodore Ts'o ex->fe_len, ex->fe_logical); 160843c73221STheodore Ts'o ex->fe_len = 0; 160943c73221STheodore Ts'o ex->fe_start = 0; 161043c73221STheodore Ts'o ex->fe_group = 0; 161143c73221STheodore Ts'o } 1612c9de560dSAlex Tomas return ex->fe_len; 1613c9de560dSAlex Tomas } 1614c9de560dSAlex Tomas 1615c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1616c9de560dSAlex Tomas { 1617c9de560dSAlex Tomas int ord; 1618c9de560dSAlex Tomas int mlen = 0; 1619c9de560dSAlex Tomas int max = 0; 1620c9de560dSAlex Tomas int cur; 1621c9de560dSAlex Tomas int start = ex->fe_start; 1622c9de560dSAlex Tomas int len = ex->fe_len; 1623c9de560dSAlex Tomas unsigned ret = 0; 1624c9de560dSAlex Tomas int len0 = len; 1625c9de560dSAlex Tomas void *buddy; 1626c9de560dSAlex Tomas 1627c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1628c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1629bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1630c9de560dSAlex Tomas mb_check_buddy(e4b); 1631c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1632c9de560dSAlex Tomas 163307b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 1634c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1635c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1636c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1637c9de560dSAlex Tomas 1638c9de560dSAlex Tomas /* let's maintain fragments counter */ 1639c9de560dSAlex Tomas if (start != 0) 1640c5e8f3f3STheodore Ts'o mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); 1641c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1642c5e8f3f3STheodore Ts'o max = !mb_test_bit(start + len, e4b->bd_bitmap); 1643c9de560dSAlex Tomas if (mlen && max) 1644c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1645c9de560dSAlex Tomas else if (!mlen && !max) 1646c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1647c9de560dSAlex Tomas 1648c9de560dSAlex Tomas /* let's maintain buddy itself */ 1649c9de560dSAlex Tomas while (len) { 1650c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1651c9de560dSAlex Tomas 1652c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1653c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1654c9de560dSAlex Tomas mlen = 1 << ord; 1655c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1656c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1657c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1658c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1659c9de560dSAlex Tomas start += mlen; 1660c9de560dSAlex Tomas len -= mlen; 1661c9de560dSAlex Tomas BUG_ON(len < 0); 1662c9de560dSAlex Tomas continue; 1663c9de560dSAlex Tomas } 1664c9de560dSAlex Tomas 1665c9de560dSAlex Tomas /* store for history */ 1666c9de560dSAlex Tomas if (ret == 0) 1667c9de560dSAlex Tomas ret = len | (ord << 16); 1668c9de560dSAlex Tomas 1669c9de560dSAlex Tomas /* we have to split large buddy */ 1670c9de560dSAlex Tomas BUG_ON(ord <= 0); 1671c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1672c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1673c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1674c9de560dSAlex Tomas 1675c9de560dSAlex Tomas ord--; 1676c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1677c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1678c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1679c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1680c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1681c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1682c9de560dSAlex Tomas } 16838a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 1684c9de560dSAlex Tomas 1685c5e8f3f3STheodore Ts'o ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0); 1686c9de560dSAlex Tomas mb_check_buddy(e4b); 1687c9de560dSAlex Tomas 1688c9de560dSAlex Tomas return ret; 1689c9de560dSAlex Tomas } 1690c9de560dSAlex Tomas 1691c9de560dSAlex Tomas /* 1692c9de560dSAlex Tomas * Must be called under group lock! 1693c9de560dSAlex Tomas */ 1694c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1695c9de560dSAlex Tomas struct ext4_buddy *e4b) 1696c9de560dSAlex Tomas { 1697c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1698c9de560dSAlex Tomas int ret; 1699c9de560dSAlex Tomas 1700c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1701c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1702c9de560dSAlex Tomas 1703c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1704c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1705c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1706c9de560dSAlex Tomas 1707c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1708c9de560dSAlex Tomas * allocated blocks for history */ 1709c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1710c9de560dSAlex Tomas 1711c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1712c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1713c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1714c9de560dSAlex Tomas 1715c3a326a6SAneesh Kumar K.V /* 1716c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 1717c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 1718c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 1719c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 1720c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 1721c3a326a6SAneesh Kumar K.V */ 1722c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1723c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1724c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1725c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 1726c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 17274ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 1728c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1729c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1730c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1731c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1732c9de560dSAlex Tomas } 173353f86b17SRitesh Harjani /* 173453f86b17SRitesh Harjani * As we've just preallocated more space than 173553f86b17SRitesh Harjani * user requested originally, we store allocated 173653f86b17SRitesh Harjani * space in a special descriptor. 173753f86b17SRitesh Harjani */ 173853f86b17SRitesh Harjani if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 173953f86b17SRitesh Harjani ext4_mb_new_preallocation(ac); 174053f86b17SRitesh Harjani 1741c9de560dSAlex Tomas } 1742c9de560dSAlex Tomas 1743c9de560dSAlex Tomas /* 1744c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1745c9de560dSAlex Tomas */ 1746c9de560dSAlex Tomas 1747c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1748c9de560dSAlex Tomas struct ext4_buddy *e4b, 1749c9de560dSAlex Tomas int finish_group) 1750c9de560dSAlex Tomas { 1751c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1752c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1753c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1754c9de560dSAlex Tomas struct ext4_free_extent ex; 1755c9de560dSAlex Tomas int max; 1756c9de560dSAlex Tomas 1757032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 1758032115fcSAneesh Kumar K.V return; 1759c9de560dSAlex Tomas /* 1760c9de560dSAlex Tomas * We don't want to scan for a whole year 1761c9de560dSAlex Tomas */ 1762c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1763c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1764c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1765c9de560dSAlex Tomas return; 1766c9de560dSAlex Tomas } 1767c9de560dSAlex Tomas 1768c9de560dSAlex Tomas /* 1769c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1770c9de560dSAlex Tomas */ 1771c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1772c9de560dSAlex Tomas return; 1773c9de560dSAlex Tomas 1774c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1775c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1776c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1777c9de560dSAlex Tomas * when it was found (within this lock-unlock 1778c9de560dSAlex Tomas * period or not) */ 177915c006a2SRobin Dong max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex); 1780c9de560dSAlex Tomas if (max >= gex->fe_len) { 1781c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1782c9de560dSAlex Tomas return; 1783c9de560dSAlex Tomas } 1784c9de560dSAlex Tomas } 1785c9de560dSAlex Tomas } 1786c9de560dSAlex Tomas 1787c9de560dSAlex Tomas /* 1788c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1789c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1790c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1791c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1792c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1793c9de560dSAlex Tomas * mballoc can't find good enough extent. 1794c9de560dSAlex Tomas * 1795c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1796c9de560dSAlex Tomas */ 1797c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1798c9de560dSAlex Tomas struct ext4_free_extent *ex, 1799c9de560dSAlex Tomas struct ext4_buddy *e4b) 1800c9de560dSAlex Tomas { 1801c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1802c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1803c9de560dSAlex Tomas 1804c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 18057137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 18067137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 1807c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1808c9de560dSAlex Tomas 1809c9de560dSAlex Tomas ac->ac_found++; 1810c9de560dSAlex Tomas 1811c9de560dSAlex Tomas /* 1812c9de560dSAlex Tomas * The special case - take what you catch first 1813c9de560dSAlex Tomas */ 1814c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1815c9de560dSAlex Tomas *bex = *ex; 1816c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1817c9de560dSAlex Tomas return; 1818c9de560dSAlex Tomas } 1819c9de560dSAlex Tomas 1820c9de560dSAlex Tomas /* 1821c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1822c9de560dSAlex Tomas */ 1823c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1824c9de560dSAlex Tomas *bex = *ex; 1825c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1826c9de560dSAlex Tomas return; 1827c9de560dSAlex Tomas } 1828c9de560dSAlex Tomas 1829c9de560dSAlex Tomas /* 1830c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1831c9de560dSAlex Tomas */ 1832c9de560dSAlex Tomas if (bex->fe_len == 0) { 1833c9de560dSAlex Tomas *bex = *ex; 1834c9de560dSAlex Tomas return; 1835c9de560dSAlex Tomas } 1836c9de560dSAlex Tomas 1837c9de560dSAlex Tomas /* 1838c9de560dSAlex Tomas * If new found extent is better, store it in the context 1839c9de560dSAlex Tomas */ 1840c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1841c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1842c9de560dSAlex Tomas * larger than previous best one is better */ 1843c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1844c9de560dSAlex Tomas *bex = *ex; 1845c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1846c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1847c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1848c9de560dSAlex Tomas * smaller than previous one */ 1849c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1850c9de560dSAlex Tomas *bex = *ex; 1851c9de560dSAlex Tomas } 1852c9de560dSAlex Tomas 1853c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1854c9de560dSAlex Tomas } 1855c9de560dSAlex Tomas 1856089ceeccSEric Sandeen static noinline_for_stack 1857089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1858c9de560dSAlex Tomas struct ext4_buddy *e4b) 1859c9de560dSAlex Tomas { 1860c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1861c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1862c9de560dSAlex Tomas int max; 1863c9de560dSAlex Tomas int err; 1864c9de560dSAlex Tomas 1865c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1866c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1867c9de560dSAlex Tomas if (err) 1868c9de560dSAlex Tomas return err; 1869c9de560dSAlex Tomas 1870c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 187115c006a2SRobin Dong max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); 1872c9de560dSAlex Tomas 1873c9de560dSAlex Tomas if (max > 0) { 1874c9de560dSAlex Tomas ac->ac_b_ex = ex; 1875c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1876c9de560dSAlex Tomas } 1877c9de560dSAlex Tomas 1878c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1879e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1880c9de560dSAlex Tomas 1881c9de560dSAlex Tomas return 0; 1882c9de560dSAlex Tomas } 1883c9de560dSAlex Tomas 1884089ceeccSEric Sandeen static noinline_for_stack 1885089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1886c9de560dSAlex Tomas struct ext4_buddy *e4b) 1887c9de560dSAlex Tomas { 1888c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1889c9de560dSAlex Tomas int max; 1890c9de560dSAlex Tomas int err; 1891c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1892838cd0cfSYongqiang Yang struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1893c9de560dSAlex Tomas struct ext4_free_extent ex; 1894c9de560dSAlex Tomas 1895c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1896c9de560dSAlex Tomas return 0; 1897838cd0cfSYongqiang Yang if (grp->bb_free == 0) 1898838cd0cfSYongqiang Yang return 0; 1899c9de560dSAlex Tomas 1900c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1901c9de560dSAlex Tomas if (err) 1902c9de560dSAlex Tomas return err; 1903c9de560dSAlex Tomas 1904163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { 1905163a203dSDarrick J. Wong ext4_mb_unload_buddy(e4b); 1906163a203dSDarrick J. Wong return 0; 1907163a203dSDarrick J. Wong } 1908163a203dSDarrick J. Wong 1909c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 191015c006a2SRobin Dong max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, 1911c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1912ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADFA11; /* debug value */ 1913c9de560dSAlex Tomas 1914c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1915c9de560dSAlex Tomas ext4_fsblk_t start; 1916c9de560dSAlex Tomas 19175661bd68SAkinobu Mita start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + 19185661bd68SAkinobu Mita ex.fe_start; 1919c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1920c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1921c9de560dSAlex Tomas ac->ac_found++; 1922c9de560dSAlex Tomas ac->ac_b_ex = ex; 1923c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1924c9de560dSAlex Tomas } 1925c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1926c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1927c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1928c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1929c9de560dSAlex Tomas ac->ac_found++; 1930c9de560dSAlex Tomas ac->ac_b_ex = ex; 1931c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1932c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1933c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1934c9de560dSAlex Tomas * number of blocks to an existing extent */ 1935c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1936c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1937c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1938c9de560dSAlex Tomas ac->ac_found++; 1939c9de560dSAlex Tomas ac->ac_b_ex = ex; 1940c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1941c9de560dSAlex Tomas } 1942c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1943e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1944c9de560dSAlex Tomas 1945c9de560dSAlex Tomas return 0; 1946c9de560dSAlex Tomas } 1947c9de560dSAlex Tomas 1948c9de560dSAlex Tomas /* 1949c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1950c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1951c9de560dSAlex Tomas */ 1952089ceeccSEric Sandeen static noinline_for_stack 1953089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1954c9de560dSAlex Tomas struct ext4_buddy *e4b) 1955c9de560dSAlex Tomas { 1956c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1957c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1958c9de560dSAlex Tomas void *buddy; 1959c9de560dSAlex Tomas int i; 1960c9de560dSAlex Tomas int k; 1961c9de560dSAlex Tomas int max; 1962c9de560dSAlex Tomas 1963c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1964c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1965c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1966c9de560dSAlex Tomas continue; 1967c9de560dSAlex Tomas 1968c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1969c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1970c9de560dSAlex Tomas 1971ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 1972eb576086SDmitry Monakhov if (k >= max) { 1973eb576086SDmitry Monakhov ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, 1974eb576086SDmitry Monakhov "%d free clusters of order %d. But found 0", 1975eb576086SDmitry Monakhov grp->bb_counters[i], i); 1976eb576086SDmitry Monakhov ext4_mark_group_bitmap_corrupted(ac->ac_sb, 1977eb576086SDmitry Monakhov e4b->bd_group, 1978eb576086SDmitry Monakhov EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1979eb576086SDmitry Monakhov break; 1980eb576086SDmitry Monakhov } 1981c9de560dSAlex Tomas ac->ac_found++; 1982c9de560dSAlex Tomas 1983c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1984c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1985c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1986c9de560dSAlex Tomas 1987c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1988c9de560dSAlex Tomas 198953f86b17SRitesh Harjani BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); 1990c9de560dSAlex Tomas 1991c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1992c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1993c9de560dSAlex Tomas 1994c9de560dSAlex Tomas break; 1995c9de560dSAlex Tomas } 1996c9de560dSAlex Tomas } 1997c9de560dSAlex Tomas 1998c9de560dSAlex Tomas /* 1999c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 2000c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 2001c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 2002c9de560dSAlex Tomas */ 2003089ceeccSEric Sandeen static noinline_for_stack 2004089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 2005c9de560dSAlex Tomas struct ext4_buddy *e4b) 2006c9de560dSAlex Tomas { 2007c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2008c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2009c9de560dSAlex Tomas struct ext4_free_extent ex; 2010c9de560dSAlex Tomas int i; 2011c9de560dSAlex Tomas int free; 2012c9de560dSAlex Tomas 2013c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 2014907ea529STheodore Ts'o if (WARN_ON(free <= 0)) 2015907ea529STheodore Ts'o return; 2016c9de560dSAlex Tomas 2017c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 2018c9de560dSAlex Tomas 2019c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 2020ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 20217137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 20227137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 202326346ff6SAneesh Kumar K.V /* 2024e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 202526346ff6SAneesh Kumar K.V * free blocks even though group info says we 202626346ff6SAneesh Kumar K.V * we have free blocks 202726346ff6SAneesh Kumar K.V */ 2028e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 202953accfa9STheodore Ts'o "%d free clusters as per " 2030fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 203126346ff6SAneesh Kumar K.V free); 2032736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2033736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2034c9de560dSAlex Tomas break; 2035c9de560dSAlex Tomas } 2036c9de560dSAlex Tomas 203715c006a2SRobin Dong mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); 2038907ea529STheodore Ts'o if (WARN_ON(ex.fe_len <= 0)) 2039907ea529STheodore Ts'o break; 204026346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 2041e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 204253accfa9STheodore Ts'o "%d free clusters as per " 2043fde4d95aSTheodore Ts'o "group info. But got %d blocks", 204426346ff6SAneesh Kumar K.V free, ex.fe_len); 2045736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2046736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2047e56eb659SAneesh Kumar K.V /* 2048e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 2049e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 2050e56eb659SAneesh Kumar K.V * without claiming the space. 2051e56eb659SAneesh Kumar K.V */ 2052e56eb659SAneesh Kumar K.V break; 205326346ff6SAneesh Kumar K.V } 2054ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADC0DE; /* debug value */ 2055c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 2056c9de560dSAlex Tomas 2057c9de560dSAlex Tomas i += ex.fe_len; 2058c9de560dSAlex Tomas free -= ex.fe_len; 2059c9de560dSAlex Tomas } 2060c9de560dSAlex Tomas 2061c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 2062c9de560dSAlex Tomas } 2063c9de560dSAlex Tomas 2064c9de560dSAlex Tomas /* 2065c9de560dSAlex Tomas * This is a special case for storages like raid5 2066506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 2067c9de560dSAlex Tomas */ 2068089ceeccSEric Sandeen static noinline_for_stack 2069089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 2070c9de560dSAlex Tomas struct ext4_buddy *e4b) 2071c9de560dSAlex Tomas { 2072c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2073c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2074c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2075c9de560dSAlex Tomas struct ext4_free_extent ex; 2076c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 2077c9de560dSAlex Tomas ext4_fsblk_t a; 2078c9de560dSAlex Tomas ext4_grpblk_t i; 2079c9de560dSAlex Tomas int max; 2080c9de560dSAlex Tomas 2081c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 2082c9de560dSAlex Tomas 2083c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 20845661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 20855661bd68SAkinobu Mita 2086c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 2087c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 2088c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 2089c9de560dSAlex Tomas 20907137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 2091c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 209215c006a2SRobin Dong max = mb_find_extent(e4b, i, sbi->s_stripe, &ex); 2093c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 2094c9de560dSAlex Tomas ac->ac_found++; 2095ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADF00D; /* debug value */ 2096c9de560dSAlex Tomas ac->ac_b_ex = ex; 2097c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2098c9de560dSAlex Tomas break; 2099c9de560dSAlex Tomas } 2100c9de560dSAlex Tomas } 2101c9de560dSAlex Tomas i += sbi->s_stripe; 2102c9de560dSAlex Tomas } 2103c9de560dSAlex Tomas } 2104c9de560dSAlex Tomas 210542ac1848SLukas Czerner /* 21068ef123feSRitesh Harjani * This is also called BEFORE we load the buddy bitmap. 210742ac1848SLukas Czerner * Returns either 1 or 0 indicating that the group is either suitable 21088ef123feSRitesh Harjani * for the allocation or not. 210942ac1848SLukas Czerner */ 21108ef123feSRitesh Harjani static bool ext4_mb_good_group(struct ext4_allocation_context *ac, 2111c9de560dSAlex Tomas ext4_group_t group, int cr) 2112c9de560dSAlex Tomas { 21138ef123feSRitesh Harjani ext4_grpblk_t free, fragments; 2114a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 2115c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2116c9de560dSAlex Tomas 2117c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 21188a57d9d6SCurt Wohlgemuth 211901fc48e8STheodore Ts'o free = grp->bb_free; 212001fc48e8STheodore Ts'o if (free == 0) 21218ef123feSRitesh Harjani return false; 212201fc48e8STheodore Ts'o if (cr <= 2 && free < ac->ac_g_ex.fe_len) 21238ef123feSRitesh Harjani return false; 212401fc48e8STheodore Ts'o 2125163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 21268ef123feSRitesh Harjani return false; 2127c9de560dSAlex Tomas 2128c9de560dSAlex Tomas fragments = grp->bb_fragments; 2129c9de560dSAlex Tomas if (fragments == 0) 21308ef123feSRitesh Harjani return false; 2131c9de560dSAlex Tomas 2132c9de560dSAlex Tomas switch (cr) { 2133c9de560dSAlex Tomas case 0: 2134c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 2135c9de560dSAlex Tomas 2136a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 2137a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 2138a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 2139a4912123STheodore Ts'o ((group % flex_size) == 0)) 21408ef123feSRitesh Harjani return false; 2141a4912123STheodore Ts'o 214240ae3487STheodore Ts'o if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) || 214340ae3487STheodore Ts'o (free / fragments) >= ac->ac_g_ex.fe_len) 21448ef123feSRitesh Harjani return true; 214540ae3487STheodore Ts'o 214640ae3487STheodore Ts'o if (grp->bb_largest_free_order < ac->ac_2order) 21478ef123feSRitesh Harjani return false; 214840ae3487STheodore Ts'o 21498ef123feSRitesh Harjani return true; 2150c9de560dSAlex Tomas case 1: 2151c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 21528ef123feSRitesh Harjani return true; 2153c9de560dSAlex Tomas break; 2154c9de560dSAlex Tomas case 2: 2155c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 21568ef123feSRitesh Harjani return true; 2157c9de560dSAlex Tomas break; 2158c9de560dSAlex Tomas case 3: 21598ef123feSRitesh Harjani return true; 2160c9de560dSAlex Tomas default: 2161c9de560dSAlex Tomas BUG(); 2162c9de560dSAlex Tomas } 2163c9de560dSAlex Tomas 21648ef123feSRitesh Harjani return false; 21658ef123feSRitesh Harjani } 21668ef123feSRitesh Harjani 21678ef123feSRitesh Harjani /* 21688ef123feSRitesh Harjani * This could return negative error code if something goes wrong 21698ef123feSRitesh Harjani * during ext4_mb_init_group(). This should not be called with 21708ef123feSRitesh Harjani * ext4_lock_group() held. 21718ef123feSRitesh Harjani */ 21728ef123feSRitesh Harjani static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, 21738ef123feSRitesh Harjani ext4_group_t group, int cr) 21748ef123feSRitesh Harjani { 21758ef123feSRitesh Harjani struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 217699377830SRitesh Harjani struct super_block *sb = ac->ac_sb; 2177c1d2c7d4SAlex Zhuravlev struct ext4_sb_info *sbi = EXT4_SB(sb); 217899377830SRitesh Harjani bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; 21798ef123feSRitesh Harjani ext4_grpblk_t free; 21808ef123feSRitesh Harjani int ret = 0; 21818ef123feSRitesh Harjani 218299377830SRitesh Harjani if (should_lock) 218399377830SRitesh Harjani ext4_lock_group(sb, group); 21848ef123feSRitesh Harjani free = grp->bb_free; 21858ef123feSRitesh Harjani if (free == 0) 21868ef123feSRitesh Harjani goto out; 21878ef123feSRitesh Harjani if (cr <= 2 && free < ac->ac_g_ex.fe_len) 21888ef123feSRitesh Harjani goto out; 21898ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 21908ef123feSRitesh Harjani goto out; 219199377830SRitesh Harjani if (should_lock) 219299377830SRitesh Harjani ext4_unlock_group(sb, group); 21938ef123feSRitesh Harjani 21948ef123feSRitesh Harjani /* We only do this if the grp has never been initialized */ 21958ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 2196c1d2c7d4SAlex Zhuravlev struct ext4_group_desc *gdp = 2197c1d2c7d4SAlex Zhuravlev ext4_get_group_desc(sb, group, NULL); 2198c1d2c7d4SAlex Zhuravlev int ret; 2199c1d2c7d4SAlex Zhuravlev 2200c1d2c7d4SAlex Zhuravlev /* cr=0/1 is a very optimistic search to find large 2201c1d2c7d4SAlex Zhuravlev * good chunks almost for free. If buddy data is not 2202c1d2c7d4SAlex Zhuravlev * ready, then this optimization makes no sense. But 2203c1d2c7d4SAlex Zhuravlev * we never skip the first block group in a flex_bg, 2204c1d2c7d4SAlex Zhuravlev * since this gets used for metadata block allocation, 2205c1d2c7d4SAlex Zhuravlev * and we want to make sure we locate metadata blocks 2206c1d2c7d4SAlex Zhuravlev * in the first block group in the flex_bg if possible. 2207c1d2c7d4SAlex Zhuravlev */ 2208c1d2c7d4SAlex Zhuravlev if (cr < 2 && 2209c1d2c7d4SAlex Zhuravlev (!sbi->s_log_groups_per_flex || 2210c1d2c7d4SAlex Zhuravlev ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && 2211c1d2c7d4SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2212c1d2c7d4SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) 2213c1d2c7d4SAlex Zhuravlev return 0; 2214c1d2c7d4SAlex Zhuravlev ret = ext4_mb_init_group(sb, group, GFP_NOFS); 22158ef123feSRitesh Harjani if (ret) 22168ef123feSRitesh Harjani return ret; 22178ef123feSRitesh Harjani } 22188ef123feSRitesh Harjani 221999377830SRitesh Harjani if (should_lock) 222099377830SRitesh Harjani ext4_lock_group(sb, group); 22218ef123feSRitesh Harjani ret = ext4_mb_good_group(ac, group, cr); 22228ef123feSRitesh Harjani out: 222399377830SRitesh Harjani if (should_lock) 222499377830SRitesh Harjani ext4_unlock_group(sb, group); 22258ef123feSRitesh Harjani return ret; 2226c9de560dSAlex Tomas } 2227c9de560dSAlex Tomas 2228cfd73237SAlex Zhuravlev /* 2229cfd73237SAlex Zhuravlev * Start prefetching @nr block bitmaps starting at @group. 2230cfd73237SAlex Zhuravlev * Return the next group which needs to be prefetched. 2231cfd73237SAlex Zhuravlev */ 22323d392b26STheodore Ts'o ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, 2233cfd73237SAlex Zhuravlev unsigned int nr, int *cnt) 2234cfd73237SAlex Zhuravlev { 2235cfd73237SAlex Zhuravlev ext4_group_t ngroups = ext4_get_groups_count(sb); 2236cfd73237SAlex Zhuravlev struct buffer_head *bh; 2237cfd73237SAlex Zhuravlev struct blk_plug plug; 2238cfd73237SAlex Zhuravlev 2239cfd73237SAlex Zhuravlev blk_start_plug(&plug); 2240cfd73237SAlex Zhuravlev while (nr-- > 0) { 2241cfd73237SAlex Zhuravlev struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, 2242cfd73237SAlex Zhuravlev NULL); 2243cfd73237SAlex Zhuravlev struct ext4_group_info *grp = ext4_get_group_info(sb, group); 2244cfd73237SAlex Zhuravlev 2245cfd73237SAlex Zhuravlev /* 2246cfd73237SAlex Zhuravlev * Prefetch block groups with free blocks; but don't 2247cfd73237SAlex Zhuravlev * bother if it is marked uninitialized on disk, since 2248cfd73237SAlex Zhuravlev * it won't require I/O to read. Also only try to 2249cfd73237SAlex Zhuravlev * prefetch once, so we avoid getblk() call, which can 2250cfd73237SAlex Zhuravlev * be expensive. 2251cfd73237SAlex Zhuravlev */ 2252cfd73237SAlex Zhuravlev if (!EXT4_MB_GRP_TEST_AND_SET_READ(grp) && 2253cfd73237SAlex Zhuravlev EXT4_MB_GRP_NEED_INIT(grp) && 2254cfd73237SAlex Zhuravlev ext4_free_group_clusters(sb, gdp) > 0 && 2255cfd73237SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2256cfd73237SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { 2257cfd73237SAlex Zhuravlev bh = ext4_read_block_bitmap_nowait(sb, group, true); 2258cfd73237SAlex Zhuravlev if (bh && !IS_ERR(bh)) { 2259cfd73237SAlex Zhuravlev if (!buffer_uptodate(bh) && cnt) 2260cfd73237SAlex Zhuravlev (*cnt)++; 2261cfd73237SAlex Zhuravlev brelse(bh); 2262cfd73237SAlex Zhuravlev } 2263cfd73237SAlex Zhuravlev } 2264cfd73237SAlex Zhuravlev if (++group >= ngroups) 2265cfd73237SAlex Zhuravlev group = 0; 2266cfd73237SAlex Zhuravlev } 2267cfd73237SAlex Zhuravlev blk_finish_plug(&plug); 2268cfd73237SAlex Zhuravlev return group; 2269cfd73237SAlex Zhuravlev } 2270cfd73237SAlex Zhuravlev 2271cfd73237SAlex Zhuravlev /* 2272cfd73237SAlex Zhuravlev * Prefetching reads the block bitmap into the buffer cache; but we 2273cfd73237SAlex Zhuravlev * need to make sure that the buddy bitmap in the page cache has been 2274cfd73237SAlex Zhuravlev * initialized. Note that ext4_mb_init_group() will block if the I/O 2275cfd73237SAlex Zhuravlev * is not yet completed, or indeed if it was not initiated by 2276cfd73237SAlex Zhuravlev * ext4_mb_prefetch did not start the I/O. 2277cfd73237SAlex Zhuravlev * 2278cfd73237SAlex Zhuravlev * TODO: We should actually kick off the buddy bitmap setup in a work 2279cfd73237SAlex Zhuravlev * queue when the buffer I/O is completed, so that we don't block 2280cfd73237SAlex Zhuravlev * waiting for the block allocation bitmap read to finish when 2281cfd73237SAlex Zhuravlev * ext4_mb_prefetch_fini is called from ext4_mb_regular_allocator(). 2282cfd73237SAlex Zhuravlev */ 22833d392b26STheodore Ts'o void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, 2284cfd73237SAlex Zhuravlev unsigned int nr) 2285cfd73237SAlex Zhuravlev { 2286cfd73237SAlex Zhuravlev while (nr-- > 0) { 2287cfd73237SAlex Zhuravlev struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, 2288cfd73237SAlex Zhuravlev NULL); 2289cfd73237SAlex Zhuravlev struct ext4_group_info *grp = ext4_get_group_info(sb, group); 2290cfd73237SAlex Zhuravlev 2291cfd73237SAlex Zhuravlev if (!group) 2292cfd73237SAlex Zhuravlev group = ext4_get_groups_count(sb); 2293cfd73237SAlex Zhuravlev group--; 2294cfd73237SAlex Zhuravlev grp = ext4_get_group_info(sb, group); 2295cfd73237SAlex Zhuravlev 2296cfd73237SAlex Zhuravlev if (EXT4_MB_GRP_NEED_INIT(grp) && 2297cfd73237SAlex Zhuravlev ext4_free_group_clusters(sb, gdp) > 0 && 2298cfd73237SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2299cfd73237SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { 2300cfd73237SAlex Zhuravlev if (ext4_mb_init_group(sb, group, GFP_NOFS)) 2301cfd73237SAlex Zhuravlev break; 2302cfd73237SAlex Zhuravlev } 2303cfd73237SAlex Zhuravlev } 2304cfd73237SAlex Zhuravlev } 2305cfd73237SAlex Zhuravlev 23064ddfef7bSEric Sandeen static noinline_for_stack int 23074ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 2308c9de560dSAlex Tomas { 2309cfd73237SAlex Zhuravlev ext4_group_t prefetch_grp = 0, ngroups, group, i; 2310bbc4ec77SRitesh Harjani int cr = -1; 231142ac1848SLukas Czerner int err = 0, first_err = 0; 2312cfd73237SAlex Zhuravlev unsigned int nr = 0, prefetch_ios = 0; 2313c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2314c9de560dSAlex Tomas struct super_block *sb; 2315c9de560dSAlex Tomas struct ext4_buddy e4b; 2316c9de560dSAlex Tomas 2317c9de560dSAlex Tomas sb = ac->ac_sb; 2318c9de560dSAlex Tomas sbi = EXT4_SB(sb); 23198df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 2320fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 232112e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 2322fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 2323fb0a387dSEric Sandeen 2324c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2325c9de560dSAlex Tomas 2326c9de560dSAlex Tomas /* first, try the goal */ 2327c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 2328c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 2329c9de560dSAlex Tomas goto out; 2330c9de560dSAlex Tomas 2331c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2332c9de560dSAlex Tomas goto out; 2333c9de560dSAlex Tomas 2334c9de560dSAlex Tomas /* 2335c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 2336c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 2337c9de560dSAlex Tomas * try exact allocation using buddy. 2338c9de560dSAlex Tomas */ 2339c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 2340c9de560dSAlex Tomas ac->ac_2order = 0; 2341c9de560dSAlex Tomas /* 2342c9de560dSAlex Tomas * We search using buddy data only if the order of the request 2343c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 2344b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 2345d9b22cf9SJan Kara * We also support searching for power-of-two requests only for 2346d9b22cf9SJan Kara * requests upto maximum buddy size we have constructed. 2347c9de560dSAlex Tomas */ 2348d9b22cf9SJan Kara if (i >= sbi->s_mb_order2_reqs && i <= sb->s_blocksize_bits + 2) { 2349c9de560dSAlex Tomas /* 2350c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2351c9de560dSAlex Tomas */ 2352c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 23531a5d5e5dSJeremy Cline ac->ac_2order = array_index_nospec(i - 1, 23541a5d5e5dSJeremy Cline sb->s_blocksize_bits + 2); 2355c9de560dSAlex Tomas } 2356c9de560dSAlex Tomas 23574ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 23584ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2359c9de560dSAlex Tomas /* TBD: may be hot point */ 2360c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2361c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2362c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2363c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2364c9de560dSAlex Tomas } 23654ba74d00STheodore Ts'o 2366c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2367c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2368c9de560dSAlex Tomas /* 2369c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2370c9de560dSAlex Tomas * cr == 3 try to get anything 2371c9de560dSAlex Tomas */ 2372c9de560dSAlex Tomas repeat: 2373c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2374c9de560dSAlex Tomas ac->ac_criteria = cr; 2375ed8f9c75SAneesh Kumar K.V /* 2376ed8f9c75SAneesh Kumar K.V * searching for the right group start 2377ed8f9c75SAneesh Kumar K.V * from the goal value specified 2378ed8f9c75SAneesh Kumar K.V */ 2379ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2380cfd73237SAlex Zhuravlev prefetch_grp = group; 2381ed8f9c75SAneesh Kumar K.V 23828df9675fSTheodore Ts'o for (i = 0; i < ngroups; group++, i++) { 238342ac1848SLukas Czerner int ret = 0; 23842ed5724dSTheodore Ts'o cond_resched(); 2385e6155736SLachlan McIlroy /* 2386e6155736SLachlan McIlroy * Artificially restricted ngroups for non-extent 2387e6155736SLachlan McIlroy * files makes group > ngroups possible on first loop. 2388e6155736SLachlan McIlroy */ 2389e6155736SLachlan McIlroy if (group >= ngroups) 2390c9de560dSAlex Tomas group = 0; 2391c9de560dSAlex Tomas 2392cfd73237SAlex Zhuravlev /* 2393cfd73237SAlex Zhuravlev * Batch reads of the block allocation bitmaps 2394cfd73237SAlex Zhuravlev * to get multiple READs in flight; limit 2395cfd73237SAlex Zhuravlev * prefetching at cr=0/1, otherwise mballoc can 2396cfd73237SAlex Zhuravlev * spend a lot of time loading imperfect groups 2397cfd73237SAlex Zhuravlev */ 2398cfd73237SAlex Zhuravlev if ((prefetch_grp == group) && 2399cfd73237SAlex Zhuravlev (cr > 1 || 2400cfd73237SAlex Zhuravlev prefetch_ios < sbi->s_mb_prefetch_limit)) { 2401cfd73237SAlex Zhuravlev unsigned int curr_ios = prefetch_ios; 2402cfd73237SAlex Zhuravlev 2403cfd73237SAlex Zhuravlev nr = sbi->s_mb_prefetch; 2404cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 2405cfd73237SAlex Zhuravlev nr = (group / sbi->s_mb_prefetch) * 2406cfd73237SAlex Zhuravlev sbi->s_mb_prefetch; 2407cfd73237SAlex Zhuravlev nr = nr + sbi->s_mb_prefetch - group; 2408cfd73237SAlex Zhuravlev } 2409cfd73237SAlex Zhuravlev prefetch_grp = ext4_mb_prefetch(sb, group, 2410cfd73237SAlex Zhuravlev nr, &prefetch_ios); 2411cfd73237SAlex Zhuravlev if (prefetch_ios == curr_ios) 2412cfd73237SAlex Zhuravlev nr = 0; 2413cfd73237SAlex Zhuravlev } 2414cfd73237SAlex Zhuravlev 24158a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 24168ef123feSRitesh Harjani ret = ext4_mb_good_group_nolock(ac, group, cr); 241742ac1848SLukas Czerner if (ret <= 0) { 241842ac1848SLukas Czerner if (!first_err) 241942ac1848SLukas Czerner first_err = ret; 2420c9de560dSAlex Tomas continue; 242142ac1848SLukas Czerner } 2422c9de560dSAlex Tomas 2423c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2424c9de560dSAlex Tomas if (err) 2425c9de560dSAlex Tomas goto out; 2426c9de560dSAlex Tomas 2427c9de560dSAlex Tomas ext4_lock_group(sb, group); 24288a57d9d6SCurt Wohlgemuth 24298a57d9d6SCurt Wohlgemuth /* 24308a57d9d6SCurt Wohlgemuth * We need to check again after locking the 24318a57d9d6SCurt Wohlgemuth * block group 24328a57d9d6SCurt Wohlgemuth */ 243342ac1848SLukas Czerner ret = ext4_mb_good_group(ac, group, cr); 24348ef123feSRitesh Harjani if (ret == 0) { 2435c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2436e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2437c9de560dSAlex Tomas continue; 2438c9de560dSAlex Tomas } 2439c9de560dSAlex Tomas 2440c9de560dSAlex Tomas ac->ac_groups_scanned++; 2441d9b22cf9SJan Kara if (cr == 0) 2442c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2443506bf2d8SEric Sandeen else if (cr == 1 && sbi->s_stripe && 2444506bf2d8SEric Sandeen !(ac->ac_g_ex.fe_len % sbi->s_stripe)) 2445c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2446c9de560dSAlex Tomas else 2447c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2448c9de560dSAlex Tomas 2449c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2450e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2451c9de560dSAlex Tomas 2452c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2453c9de560dSAlex Tomas break; 2454c9de560dSAlex Tomas } 2455c9de560dSAlex Tomas } 2456c9de560dSAlex Tomas 2457c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2458c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2459c9de560dSAlex Tomas /* 2460c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2461c9de560dSAlex Tomas * the best chunk we've found so far 2462c9de560dSAlex Tomas */ 2463c9de560dSAlex Tomas 2464c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2465c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2466c9de560dSAlex Tomas /* 2467c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2468c9de560dSAlex Tomas * The only thing we can do is just take first 2469c9de560dSAlex Tomas * found block(s) 2470c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2471c9de560dSAlex Tomas */ 2472c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2473c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2474c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2475c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2476c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2477c9de560dSAlex Tomas cr = 3; 2478c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2479c9de560dSAlex Tomas goto repeat; 2480c9de560dSAlex Tomas } 2481c9de560dSAlex Tomas } 2482c9de560dSAlex Tomas out: 248342ac1848SLukas Czerner if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) 248442ac1848SLukas Czerner err = first_err; 2485bbc4ec77SRitesh Harjani 2486d3df1453SRitesh Harjani mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n", 2487bbc4ec77SRitesh Harjani ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, 2488bbc4ec77SRitesh Harjani ac->ac_flags, cr, err); 2489cfd73237SAlex Zhuravlev 2490cfd73237SAlex Zhuravlev if (nr) 2491cfd73237SAlex Zhuravlev ext4_mb_prefetch_fini(sb, prefetch_grp, nr); 2492cfd73237SAlex Zhuravlev 2493c9de560dSAlex Tomas return err; 2494c9de560dSAlex Tomas } 2495c9de560dSAlex Tomas 2496c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2497c9de560dSAlex Tomas { 2498247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2499c9de560dSAlex Tomas ext4_group_t group; 2500c9de560dSAlex Tomas 25018df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2502c9de560dSAlex Tomas return NULL; 2503c9de560dSAlex Tomas group = *pos + 1; 2504a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2505c9de560dSAlex Tomas } 2506c9de560dSAlex Tomas 2507c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2508c9de560dSAlex Tomas { 2509247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2510c9de560dSAlex Tomas ext4_group_t group; 2511c9de560dSAlex Tomas 2512c9de560dSAlex Tomas ++*pos; 25138df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2514c9de560dSAlex Tomas return NULL; 2515c9de560dSAlex Tomas group = *pos + 1; 2516a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2517c9de560dSAlex Tomas } 2518c9de560dSAlex Tomas 2519c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2520c9de560dSAlex Tomas { 2521247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2522a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2523c9de560dSAlex Tomas int i; 25241c8457caSAditya Kali int err, buddy_loaded = 0; 2525c9de560dSAlex Tomas struct ext4_buddy e4b; 25261c8457caSAditya Kali struct ext4_group_info *grinfo; 25272df2c340SArnd Bergmann unsigned char blocksize_bits = min_t(unsigned char, 25282df2c340SArnd Bergmann sb->s_blocksize_bits, 25292df2c340SArnd Bergmann EXT4_MAX_BLOCK_LOG_SIZE); 2530c9de560dSAlex Tomas struct sg { 2531c9de560dSAlex Tomas struct ext4_group_info info; 2532b80b32b6STheodore Ts'o ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; 2533c9de560dSAlex Tomas } sg; 2534c9de560dSAlex Tomas 2535c9de560dSAlex Tomas group--; 2536c9de560dSAlex Tomas if (group == 0) 253797b4af2fSRasmus Villemoes seq_puts(seq, "#group: free frags first [" 253897b4af2fSRasmus Villemoes " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " 2539802cf1f9SHuaitong Han " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); 2540c9de560dSAlex Tomas 2541b80b32b6STheodore Ts'o i = (blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2542b80b32b6STheodore Ts'o sizeof(struct ext4_group_info); 2543b80b32b6STheodore Ts'o 25441c8457caSAditya Kali grinfo = ext4_get_group_info(sb, group); 25451c8457caSAditya Kali /* Load the group info in memory only if not already loaded. */ 25461c8457caSAditya Kali if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { 2547c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2548c9de560dSAlex Tomas if (err) { 2549a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2550c9de560dSAlex Tomas return 0; 2551c9de560dSAlex Tomas } 25521c8457caSAditya Kali buddy_loaded = 1; 25531c8457caSAditya Kali } 25541c8457caSAditya Kali 2555b80b32b6STheodore Ts'o memcpy(&sg, ext4_get_group_info(sb, group), i); 25561c8457caSAditya Kali 25571c8457caSAditya Kali if (buddy_loaded) 2558e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2559c9de560dSAlex Tomas 2560a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2561c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2562c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 25632df2c340SArnd Bergmann seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? 2564c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2565c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2566c9de560dSAlex Tomas 2567c9de560dSAlex Tomas return 0; 2568c9de560dSAlex Tomas } 2569c9de560dSAlex Tomas 2570c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2571c9de560dSAlex Tomas { 2572c9de560dSAlex Tomas } 2573c9de560dSAlex Tomas 2574247dbed8SChristoph Hellwig const struct seq_operations ext4_mb_seq_groups_ops = { 2575c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2576c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2577c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2578c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2579c9de560dSAlex Tomas }; 2580c9de560dSAlex Tomas 2581fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 2582fb1813f4SCurt Wohlgemuth { 2583fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 2584fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 2585fb1813f4SCurt Wohlgemuth 2586fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 2587fb1813f4SCurt Wohlgemuth return cachep; 2588fb1813f4SCurt Wohlgemuth } 25895f21b0e6SFrederic Bohe 259028623c2fSTheodore Ts'o /* 259128623c2fSTheodore Ts'o * Allocate the top-level s_group_info array for the specified number 259228623c2fSTheodore Ts'o * of groups 259328623c2fSTheodore Ts'o */ 259428623c2fSTheodore Ts'o int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) 259528623c2fSTheodore Ts'o { 259628623c2fSTheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 259728623c2fSTheodore Ts'o unsigned size; 2598df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***old_groupinfo, ***new_groupinfo; 259928623c2fSTheodore Ts'o 260028623c2fSTheodore Ts'o size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> 260128623c2fSTheodore Ts'o EXT4_DESC_PER_BLOCK_BITS(sb); 260228623c2fSTheodore Ts'o if (size <= sbi->s_group_info_size) 260328623c2fSTheodore Ts'o return 0; 260428623c2fSTheodore Ts'o 260528623c2fSTheodore Ts'o size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); 2606a7c3e901SMichal Hocko new_groupinfo = kvzalloc(size, GFP_KERNEL); 260728623c2fSTheodore Ts'o if (!new_groupinfo) { 260828623c2fSTheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 260928623c2fSTheodore Ts'o return -ENOMEM; 261028623c2fSTheodore Ts'o } 2611df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2612df3da4eaSSuraj Jitindar Singh old_groupinfo = rcu_dereference(sbi->s_group_info); 2613df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 2614df3da4eaSSuraj Jitindar Singh memcpy(new_groupinfo, old_groupinfo, 261528623c2fSTheodore Ts'o sbi->s_group_info_size * sizeof(*sbi->s_group_info)); 2616df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2617df3da4eaSSuraj Jitindar Singh rcu_assign_pointer(sbi->s_group_info, new_groupinfo); 261828623c2fSTheodore Ts'o sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); 2619df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 2620df3da4eaSSuraj Jitindar Singh ext4_kvfree_array_rcu(old_groupinfo); 262128623c2fSTheodore Ts'o ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", 262228623c2fSTheodore Ts'o sbi->s_group_info_size); 262328623c2fSTheodore Ts'o return 0; 262428623c2fSTheodore Ts'o } 262528623c2fSTheodore Ts'o 26265f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 2627920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 26285f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 26295f21b0e6SFrederic Bohe { 2630fb1813f4SCurt Wohlgemuth int i; 26315f21b0e6SFrederic Bohe int metalen = 0; 2632df3da4eaSSuraj Jitindar Singh int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb); 26335f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 26345f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 2635fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 26365f21b0e6SFrederic Bohe 26375f21b0e6SFrederic Bohe /* 26385f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 26395f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 26405f21b0e6SFrederic Bohe * to ext4_group_info structures 26415f21b0e6SFrederic Bohe */ 26425f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 26435f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 26445f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 26454fdb5543SDmitry Monakhov meta_group_info = kmalloc(metalen, GFP_NOFS); 26465f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 26477f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate mem " 26489d8b9ec4STheodore Ts'o "for a buddy group"); 26495f21b0e6SFrederic Bohe goto exit_meta_group_info; 26505f21b0e6SFrederic Bohe } 2651df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2652df3da4eaSSuraj Jitindar Singh rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; 2653df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 26545f21b0e6SFrederic Bohe } 26555f21b0e6SFrederic Bohe 2656df3da4eaSSuraj Jitindar Singh meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); 26575f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 26585f21b0e6SFrederic Bohe 26594fdb5543SDmitry Monakhov meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS); 26605f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 26617f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); 26625f21b0e6SFrederic Bohe goto exit_group_info; 26635f21b0e6SFrederic Bohe } 26645f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 26655f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 26665f21b0e6SFrederic Bohe 26675f21b0e6SFrederic Bohe /* 26685f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 26695f21b0e6SFrederic Bohe * empty groups without initialization 26705f21b0e6SFrederic Bohe */ 26718844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 26728844618dSTheodore Ts'o (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 26735f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2674cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, group, desc); 26755f21b0e6SFrederic Bohe } else { 26765f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2677021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, desc); 26785f21b0e6SFrederic Bohe } 26795f21b0e6SFrederic Bohe 26805f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 2681920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 268264e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 26838a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 26845f21b0e6SFrederic Bohe 2685a3450215SRitesh Harjani mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group); 26865f21b0e6SFrederic Bohe return 0; 26875f21b0e6SFrederic Bohe 26885f21b0e6SFrederic Bohe exit_group_info: 26895f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 2690caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 2691df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 2692df3da4eaSSuraj Jitindar Singh 2693df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2694df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2695df3da4eaSSuraj Jitindar Singh kfree(group_info[idx]); 2696df3da4eaSSuraj Jitindar Singh group_info[idx] = NULL; 2697df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2698caaf7a29STao Ma } 26995f21b0e6SFrederic Bohe exit_meta_group_info: 27005f21b0e6SFrederic Bohe return -ENOMEM; 27015f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 27025f21b0e6SFrederic Bohe 2703c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2704c9de560dSAlex Tomas { 27058df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2706c9de560dSAlex Tomas ext4_group_t i; 2707c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 270828623c2fSTheodore Ts'o int err; 27095f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 2710df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 2711fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 2712c9de560dSAlex Tomas 271328623c2fSTheodore Ts'o err = ext4_mb_alloc_groupinfo(sb, ngroups); 271428623c2fSTheodore Ts'o if (err) 271528623c2fSTheodore Ts'o return err; 27165f21b0e6SFrederic Bohe 2717c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2718c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 27199d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 2720c9de560dSAlex Tomas goto err_freesgi; 2721c9de560dSAlex Tomas } 272248e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 272348e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 272448e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 272548e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 272648e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 2727c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 27288df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 27294b99faa2SKhazhismel Kumykov cond_resched(); 2730c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2731c9de560dSAlex Tomas if (desc == NULL) { 27329d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 2733c9de560dSAlex Tomas goto err_freebuddy; 2734c9de560dSAlex Tomas } 27355f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 27365f21b0e6SFrederic Bohe goto err_freebuddy; 2737c9de560dSAlex Tomas } 2738c9de560dSAlex Tomas 2739cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 2740cfd73237SAlex Zhuravlev /* a single flex group is supposed to be read by a single IO */ 2741cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = 1 << sbi->s_es->s_log_groups_per_flex; 2742cfd73237SAlex Zhuravlev sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ 2743cfd73237SAlex Zhuravlev } else { 2744cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = 32; 2745cfd73237SAlex Zhuravlev } 2746cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) 2747cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = ext4_get_groups_count(sb); 2748cfd73237SAlex Zhuravlev /* now many real IOs to prefetch within a single allocation at cr=0 2749cfd73237SAlex Zhuravlev * given cr=0 is an CPU-related optimization we shouldn't try to 2750cfd73237SAlex Zhuravlev * load too many groups, at some point we should start to use what 2751cfd73237SAlex Zhuravlev * we've got in memory. 2752cfd73237SAlex Zhuravlev * with an average random access time 5ms, it'd take a second to get 2753cfd73237SAlex Zhuravlev * 200 groups (* N with flex_bg), so let's make this limit 4 2754cfd73237SAlex Zhuravlev */ 2755cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; 2756cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) 2757cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); 2758cfd73237SAlex Zhuravlev 2759c9de560dSAlex Tomas return 0; 2760c9de560dSAlex Tomas 2761c9de560dSAlex Tomas err_freebuddy: 2762fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2763f1fa3342SRoel Kluin while (i-- > 0) 2764fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, ext4_get_group_info(sb, i)); 276528623c2fSTheodore Ts'o i = sbi->s_group_info_size; 2766df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2767df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2768f1fa3342SRoel Kluin while (i-- > 0) 2769df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 2770df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2771c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2772c9de560dSAlex Tomas err_freesgi: 2773df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2774df3da4eaSSuraj Jitindar Singh kvfree(rcu_dereference(sbi->s_group_info)); 2775df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2776c9de560dSAlex Tomas return -ENOMEM; 2777c9de560dSAlex Tomas } 2778c9de560dSAlex Tomas 27792892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 27802892c15dSEric Sandeen { 27812892c15dSEric Sandeen int i; 27822892c15dSEric Sandeen 27832892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 27842892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 27852892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 27862892c15dSEric Sandeen } 27872892c15dSEric Sandeen } 27882892c15dSEric Sandeen 27892892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 27902892c15dSEric Sandeen { 27912892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 27922892c15dSEric Sandeen int slab_size; 27932892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 27942892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 27952892c15dSEric Sandeen struct kmem_cache *cachep; 27962892c15dSEric Sandeen 27972892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 27982892c15dSEric Sandeen return -EINVAL; 27992892c15dSEric Sandeen 28002892c15dSEric Sandeen if (unlikely(cache_index < 0)) 28012892c15dSEric Sandeen cache_index = 0; 28022892c15dSEric Sandeen 28032892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 28042892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 28052892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 28062892c15dSEric Sandeen return 0; /* Already created */ 28072892c15dSEric Sandeen } 28082892c15dSEric Sandeen 28092892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 28102892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 28112892c15dSEric Sandeen 28122892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 28132892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 28142892c15dSEric Sandeen NULL); 28152892c15dSEric Sandeen 2816823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 2817823ba01fSTao Ma 28182892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 28192892c15dSEric Sandeen if (!cachep) { 28209d8b9ec4STheodore Ts'o printk(KERN_EMERG 28219d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 28222892c15dSEric Sandeen return -ENOMEM; 28232892c15dSEric Sandeen } 28242892c15dSEric Sandeen 28252892c15dSEric Sandeen return 0; 28262892c15dSEric Sandeen } 28272892c15dSEric Sandeen 28289d99012fSAkira Fujita int ext4_mb_init(struct super_block *sb) 2829c9de560dSAlex Tomas { 2830c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 28316be2ded1SAneesh Kumar K.V unsigned i, j; 2832935244cdSNicolai Stange unsigned offset, offset_incr; 2833c9de560dSAlex Tomas unsigned max; 283474767c5aSShen Feng int ret; 2835c9de560dSAlex Tomas 28361927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); 2837c9de560dSAlex Tomas 2838c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2839c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2840fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2841fb1813f4SCurt Wohlgemuth goto out; 2842c9de560dSAlex Tomas } 2843ff7ef329SYasunori Goto 28441927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); 2845c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2846c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2847fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2848fb1813f4SCurt Wohlgemuth goto out; 2849fb1813f4SCurt Wohlgemuth } 2850fb1813f4SCurt Wohlgemuth 28512892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 28522892c15dSEric Sandeen if (ret < 0) 2853fb1813f4SCurt Wohlgemuth goto out; 2854c9de560dSAlex Tomas 2855c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2856c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2857c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2858c9de560dSAlex Tomas 2859c9de560dSAlex Tomas i = 1; 2860c9de560dSAlex Tomas offset = 0; 2861935244cdSNicolai Stange offset_incr = 1 << (sb->s_blocksize_bits - 1); 2862c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2863c9de560dSAlex Tomas do { 2864c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2865c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2866935244cdSNicolai Stange offset += offset_incr; 2867935244cdSNicolai Stange offset_incr = offset_incr >> 1; 2868c9de560dSAlex Tomas max = max >> 1; 2869c9de560dSAlex Tomas i++; 2870c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2871c9de560dSAlex Tomas 2872c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2873c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2874d08854f5STheodore Ts'o sbi->s_mb_free_pending = 0; 2875a0154344SDaeho Jeong INIT_LIST_HEAD(&sbi->s_freed_data_list); 2876c9de560dSAlex Tomas 2877c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2878c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2879c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2880c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2881c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 288227baebb8STheodore Ts'o /* 288327baebb8STheodore Ts'o * The default group preallocation is 512, which for 4k block 288427baebb8STheodore Ts'o * sizes translates to 2 megabytes. However for bigalloc file 288527baebb8STheodore Ts'o * systems, this is probably too big (i.e, if the cluster size 288627baebb8STheodore Ts'o * is 1 megabyte, then group preallocation size becomes half a 288727baebb8STheodore Ts'o * gigabyte!). As a default, we will keep a two megabyte 288827baebb8STheodore Ts'o * group pralloc size for cluster sizes up to 64k, and after 288927baebb8STheodore Ts'o * that, we will force a minimum group preallocation size of 289027baebb8STheodore Ts'o * 32 clusters. This translates to 8 megs when the cluster 289127baebb8STheodore Ts'o * size is 256k, and 32 megs when the cluster size is 1 meg, 289227baebb8STheodore Ts'o * which seems reasonable as a default. 289327baebb8STheodore Ts'o */ 289427baebb8STheodore Ts'o sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> 289527baebb8STheodore Ts'o sbi->s_cluster_bits, 32); 2896d7a1fee1SDan Ehrenberg /* 2897d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 2898d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 2899d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 2900d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 2901d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 2902d7a1fee1SDan Ehrenberg * the stripes. 2903d7a1fee1SDan Ehrenberg */ 2904d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 2905d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 2906d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc, sbi->s_stripe); 2907d7a1fee1SDan Ehrenberg } 2908c9de560dSAlex Tomas 2909730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 2910c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2911fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2912029b10c5SAndrey Tsyvarev goto out; 2913c9de560dSAlex Tomas } 2914730c213cSEric Sandeen for_each_possible_cpu(i) { 2915c9de560dSAlex Tomas struct ext4_locality_group *lg; 2916730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 2917c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 29186be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 29196be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 2920c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2921c9de560dSAlex Tomas } 2922c9de560dSAlex Tomas 292379a77c5aSYu Jian /* init file for buddy data */ 292479a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 29257aa0baeaSTao Ma if (ret != 0) 29267aa0baeaSTao Ma goto out_free_locality_groups; 292779a77c5aSYu Jian 29287aa0baeaSTao Ma return 0; 29297aa0baeaSTao Ma 29307aa0baeaSTao Ma out_free_locality_groups: 29317aa0baeaSTao Ma free_percpu(sbi->s_locality_groups); 29327aa0baeaSTao Ma sbi->s_locality_groups = NULL; 2933fb1813f4SCurt Wohlgemuth out: 2934fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 29357aa0baeaSTao Ma sbi->s_mb_offsets = NULL; 2936fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 29377aa0baeaSTao Ma sbi->s_mb_maxs = NULL; 2938fb1813f4SCurt Wohlgemuth return ret; 2939c9de560dSAlex Tomas } 2940c9de560dSAlex Tomas 2941955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 2942d3df1453SRitesh Harjani static int ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2943c9de560dSAlex Tomas { 2944c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2945c9de560dSAlex Tomas struct list_head *cur, *tmp; 2946c9de560dSAlex Tomas int count = 0; 2947c9de560dSAlex Tomas 2948c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2949c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2950c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2951c9de560dSAlex Tomas count++; 2952688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 2953c9de560dSAlex Tomas } 2954d3df1453SRitesh Harjani return count; 2955c9de560dSAlex Tomas } 2956c9de560dSAlex Tomas 2957c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2958c9de560dSAlex Tomas { 29598df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2960c9de560dSAlex Tomas ext4_group_t i; 2961c9de560dSAlex Tomas int num_meta_group_infos; 2962df3da4eaSSuraj Jitindar Singh struct ext4_group_info *grinfo, ***group_info; 2963c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2964fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2965d3df1453SRitesh Harjani int count; 2966c9de560dSAlex Tomas 2967c9de560dSAlex Tomas if (sbi->s_group_info) { 29688df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 29694b99faa2SKhazhismel Kumykov cond_resched(); 2970c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2971a3450215SRitesh Harjani mb_group_bb_bitmap_free(grinfo); 2972c9de560dSAlex Tomas ext4_lock_group(sb, i); 2973d3df1453SRitesh Harjani count = ext4_mb_cleanup_pa(grinfo); 2974d3df1453SRitesh Harjani if (count) 2975d3df1453SRitesh Harjani mb_debug(sb, "mballoc: %d PAs left\n", 2976d3df1453SRitesh Harjani count); 2977c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2978fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 2979c9de560dSAlex Tomas } 29808df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 2981c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2982c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2983df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 2984df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 2985c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2986df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 2987df3da4eaSSuraj Jitindar Singh kvfree(group_info); 2988df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 2989c9de560dSAlex Tomas } 2990c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2991c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2992c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2993c9de560dSAlex Tomas if (sbi->s_mb_stats) { 29949d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 29959d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 2996c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2997c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2998c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 29999d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 30009d8b9ec4STheodore Ts'o "mballoc: %u extents scanned, %u goal hits, " 30019d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 3002c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 3003c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 3004c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 3005c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 3006c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 30079d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 30089d8b9ec4STheodore Ts'o "mballoc: %lu generated and it took %Lu", 3009ced156e4STao Ma sbi->s_mb_buddies_generated, 3010c9de560dSAlex Tomas sbi->s_mb_generation_time); 30119d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 30129d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 3013c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 3014c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 3015c9de560dSAlex Tomas } 3016c9de560dSAlex Tomas 3017730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 3018c9de560dSAlex Tomas 3019c9de560dSAlex Tomas return 0; 3020c9de560dSAlex Tomas } 3021c9de560dSAlex Tomas 302277ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 3023a0154344SDaeho Jeong ext4_group_t block_group, ext4_grpblk_t cluster, int count, 3024a0154344SDaeho Jeong struct bio **biop) 30255c521830SJiaying Zhang { 30265c521830SJiaying Zhang ext4_fsblk_t discard_block; 30275c521830SJiaying Zhang 302884130193STheodore Ts'o discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + 302984130193STheodore Ts'o ext4_group_first_block_no(sb, block_group)); 303084130193STheodore Ts'o count = EXT4_C2B(EXT4_SB(sb), count); 30315c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 30325c521830SJiaying Zhang (unsigned long long) discard_block, count); 3033a0154344SDaeho Jeong if (biop) { 3034a0154344SDaeho Jeong return __blkdev_issue_discard(sb->s_bdev, 3035a0154344SDaeho Jeong (sector_t)discard_block << (sb->s_blocksize_bits - 9), 3036a0154344SDaeho Jeong (sector_t)count << (sb->s_blocksize_bits - 9), 3037a0154344SDaeho Jeong GFP_NOFS, 0, biop); 3038a0154344SDaeho Jeong } else 303993259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 30405c521830SJiaying Zhang } 30415c521830SJiaying Zhang 3042a0154344SDaeho Jeong static void ext4_free_data_in_buddy(struct super_block *sb, 3043a0154344SDaeho Jeong struct ext4_free_data *entry) 3044c9de560dSAlex Tomas { 3045c9de560dSAlex Tomas struct ext4_buddy e4b; 3046c894058dSAneesh Kumar K.V struct ext4_group_info *db; 3047d9f34504STheodore Ts'o int err, count = 0, count2 = 0; 3048c9de560dSAlex Tomas 3049d3df1453SRitesh Harjani mb_debug(sb, "gonna free %u blocks in group %u (0x%p):", 305018aadd47SBobi Jam entry->efd_count, entry->efd_group, entry); 3051c9de560dSAlex Tomas 305218aadd47SBobi Jam err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); 3053c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 3054c9de560dSAlex Tomas BUG_ON(err != 0); 3055c9de560dSAlex Tomas 3056d08854f5STheodore Ts'o spin_lock(&EXT4_SB(sb)->s_md_lock); 3057d08854f5STheodore Ts'o EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count; 3058d08854f5STheodore Ts'o spin_unlock(&EXT4_SB(sb)->s_md_lock); 305918aadd47SBobi Jam 3060c894058dSAneesh Kumar K.V db = e4b.bd_info; 3061c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 306218aadd47SBobi Jam count += entry->efd_count; 3063c9de560dSAlex Tomas count2++; 306418aadd47SBobi Jam ext4_lock_group(sb, entry->efd_group); 3065c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 306618aadd47SBobi Jam rb_erase(&entry->efd_node, &(db->bb_free_root)); 306718aadd47SBobi Jam mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); 3068c9de560dSAlex Tomas 30693d56b8d2STao Ma /* 30703d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 30713d56b8d2STao Ma * ext4_trim_fs can trim it. 30723d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 30733d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 30743d56b8d2STao Ma */ 30753d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 30763d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 30773d56b8d2STao Ma 3078c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 3079c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 3080c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 3081c894058dSAneesh Kumar K.V */ 308209cbfeafSKirill A. Shutemov put_page(e4b.bd_buddy_page); 308309cbfeafSKirill A. Shutemov put_page(e4b.bd_bitmap_page); 3084c894058dSAneesh Kumar K.V } 308518aadd47SBobi Jam ext4_unlock_group(sb, entry->efd_group); 308618aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 3087e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3088c9de560dSAlex Tomas 3089d3df1453SRitesh Harjani mb_debug(sb, "freed %d blocks in %d structures\n", count, 3090d3df1453SRitesh Harjani count2); 3091c9de560dSAlex Tomas } 3092c9de560dSAlex Tomas 3093a0154344SDaeho Jeong /* 3094a0154344SDaeho Jeong * This function is called by the jbd2 layer once the commit has finished, 3095a0154344SDaeho Jeong * so we know we can free the blocks that were released with that commit. 3096a0154344SDaeho Jeong */ 3097a0154344SDaeho Jeong void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid) 3098a0154344SDaeho Jeong { 3099a0154344SDaeho Jeong struct ext4_sb_info *sbi = EXT4_SB(sb); 3100a0154344SDaeho Jeong struct ext4_free_data *entry, *tmp; 3101a0154344SDaeho Jeong struct bio *discard_bio = NULL; 3102a0154344SDaeho Jeong struct list_head freed_data_list; 3103a0154344SDaeho Jeong struct list_head *cut_pos = NULL; 3104a0154344SDaeho Jeong int err; 3105a0154344SDaeho Jeong 3106a0154344SDaeho Jeong INIT_LIST_HEAD(&freed_data_list); 3107a0154344SDaeho Jeong 3108a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 3109a0154344SDaeho Jeong list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) { 3110a0154344SDaeho Jeong if (entry->efd_tid != commit_tid) 3111a0154344SDaeho Jeong break; 3112a0154344SDaeho Jeong cut_pos = &entry->efd_list; 3113a0154344SDaeho Jeong } 3114a0154344SDaeho Jeong if (cut_pos) 3115a0154344SDaeho Jeong list_cut_position(&freed_data_list, &sbi->s_freed_data_list, 3116a0154344SDaeho Jeong cut_pos); 3117a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 3118a0154344SDaeho Jeong 3119a0154344SDaeho Jeong if (test_opt(sb, DISCARD)) { 3120a0154344SDaeho Jeong list_for_each_entry(entry, &freed_data_list, efd_list) { 3121a0154344SDaeho Jeong err = ext4_issue_discard(sb, entry->efd_group, 3122a0154344SDaeho Jeong entry->efd_start_cluster, 3123a0154344SDaeho Jeong entry->efd_count, 3124a0154344SDaeho Jeong &discard_bio); 3125a0154344SDaeho Jeong if (err && err != -EOPNOTSUPP) { 3126a0154344SDaeho Jeong ext4_msg(sb, KERN_WARNING, "discard request in" 3127a0154344SDaeho Jeong " group:%d block:%d count:%d failed" 3128a0154344SDaeho Jeong " with %d", entry->efd_group, 3129a0154344SDaeho Jeong entry->efd_start_cluster, 3130a0154344SDaeho Jeong entry->efd_count, err); 3131a0154344SDaeho Jeong } else if (err == -EOPNOTSUPP) 3132a0154344SDaeho Jeong break; 3133a0154344SDaeho Jeong } 3134a0154344SDaeho Jeong 3135e4510577SDaeho Jeong if (discard_bio) { 3136a0154344SDaeho Jeong submit_bio_wait(discard_bio); 3137e4510577SDaeho Jeong bio_put(discard_bio); 3138e4510577SDaeho Jeong } 3139a0154344SDaeho Jeong } 3140a0154344SDaeho Jeong 3141a0154344SDaeho Jeong list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list) 3142a0154344SDaeho Jeong ext4_free_data_in_buddy(sb, entry); 3143a0154344SDaeho Jeong } 3144a0154344SDaeho Jeong 31455dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 3146c9de560dSAlex Tomas { 314716828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 314816828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3149c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 3150f283529aSRitesh Harjani goto out; 3151c9de560dSAlex Tomas 315216828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 315316828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3154f283529aSRitesh Harjani if (ext4_ac_cachep == NULL) 3155f283529aSRitesh Harjani goto out_pa_free; 3156c894058dSAneesh Kumar K.V 315718aadd47SBobi Jam ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, 315816828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3159f283529aSRitesh Harjani if (ext4_free_data_cachep == NULL) 3160f283529aSRitesh Harjani goto out_ac_free; 3161f283529aSRitesh Harjani 3162c9de560dSAlex Tomas return 0; 3163f283529aSRitesh Harjani 3164f283529aSRitesh Harjani out_ac_free: 3165f283529aSRitesh Harjani kmem_cache_destroy(ext4_ac_cachep); 3166f283529aSRitesh Harjani out_pa_free: 3167f283529aSRitesh Harjani kmem_cache_destroy(ext4_pspace_cachep); 3168f283529aSRitesh Harjani out: 3169f283529aSRitesh Harjani return -ENOMEM; 3170c9de560dSAlex Tomas } 3171c9de560dSAlex Tomas 31725dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 3173c9de560dSAlex Tomas { 31743e03f9caSJesper Dangaard Brouer /* 31753e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 31763e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 31773e03f9caSJesper Dangaard Brouer */ 31783e03f9caSJesper Dangaard Brouer rcu_barrier(); 3179c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 3180256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 318118aadd47SBobi Jam kmem_cache_destroy(ext4_free_data_cachep); 31822892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 3183c9de560dSAlex Tomas } 3184c9de560dSAlex Tomas 3185c9de560dSAlex Tomas 3186c9de560dSAlex Tomas /* 318773b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 3188c9de560dSAlex Tomas * Returns 0 if success or error code 3189c9de560dSAlex Tomas */ 31904ddfef7bSEric Sandeen static noinline_for_stack int 31914ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 319253accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 3193c9de560dSAlex Tomas { 3194c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3195c9de560dSAlex Tomas struct ext4_group_desc *gdp; 3196c9de560dSAlex Tomas struct buffer_head *gdp_bh; 3197c9de560dSAlex Tomas struct ext4_sb_info *sbi; 3198c9de560dSAlex Tomas struct super_block *sb; 3199c9de560dSAlex Tomas ext4_fsblk_t block; 3200519deca0SAneesh Kumar K.V int err, len; 3201c9de560dSAlex Tomas 3202c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3203c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 3204c9de560dSAlex Tomas 3205c9de560dSAlex Tomas sb = ac->ac_sb; 3206c9de560dSAlex Tomas sbi = EXT4_SB(sb); 3207c9de560dSAlex Tomas 3208574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 32099008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 32109008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 32119008a58eSDarrick J. Wong bitmap_bh = NULL; 3212c9de560dSAlex Tomas goto out_err; 32139008a58eSDarrick J. Wong } 3214c9de560dSAlex Tomas 32155d601255Sliang xie BUFFER_TRACE(bitmap_bh, "getting write access"); 3216c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 3217c9de560dSAlex Tomas if (err) 3218c9de560dSAlex Tomas goto out_err; 3219c9de560dSAlex Tomas 3220c9de560dSAlex Tomas err = -EIO; 3221c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 3222c9de560dSAlex Tomas if (!gdp) 3223c9de560dSAlex Tomas goto out_err; 3224c9de560dSAlex Tomas 3225a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 3226021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, gdp)); 322703cddb80SAneesh Kumar K.V 32285d601255Sliang xie BUFFER_TRACE(gdp_bh, "get_write_access"); 3229c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 3230c9de560dSAlex Tomas if (err) 3231c9de560dSAlex Tomas goto out_err; 3232c9de560dSAlex Tomas 3233bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3234c9de560dSAlex Tomas 323553accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3236ce9f24ccSJan Kara if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { 323712062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 32381084f252STheodore Ts'o "fs metadata", block, block+len); 3239519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 3240554a5cccSVegard Nossum * Fix the bitmap and return EFSCORRUPTED 3241519deca0SAneesh Kumar K.V * We leak some of the blocks here. 3242519deca0SAneesh Kumar K.V */ 3243955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3244c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3245519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 3246955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 32470390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3248519deca0SAneesh Kumar K.V if (!err) 3249554a5cccSVegard Nossum err = -EFSCORRUPTED; 3250519deca0SAneesh Kumar K.V goto out_err; 3251c9de560dSAlex Tomas } 3252955ce5f5SAneesh Kumar K.V 3253955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3254c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 3255c9de560dSAlex Tomas { 3256c9de560dSAlex Tomas int i; 3257c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 3258c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 3259c9de560dSAlex Tomas bitmap_bh->b_data)); 3260c9de560dSAlex Tomas } 3261c9de560dSAlex Tomas } 3262c9de560dSAlex Tomas #endif 3263c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3264c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 32658844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 32668844618dSTheodore Ts'o (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 3267c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 3268021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, 3269cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, 3270560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 3271c9de560dSAlex Tomas } 3272021b65bbSTheodore Ts'o len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; 3273021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, len); 327479f1ba49STao Ma ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); 3275feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); 3276955ce5f5SAneesh Kumar K.V 3277955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 327857042651STheodore Ts'o percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); 3279d2a17637SMingming Cao /* 32806bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 3281d2a17637SMingming Cao */ 32826bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 32836bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 328457042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 328557042651STheodore Ts'o reserv_clstrs); 3286c9de560dSAlex Tomas 3287772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 3288772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 3289772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 329090ba983fSTheodore Ts'o atomic64_sub(ac->ac_b_ex.fe_len, 32917c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 32927c990728SSuraj Jitindar Singh flex_group)->free_clusters); 3293772cb7c8SJose R. Santos } 3294772cb7c8SJose R. Santos 32950390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3296c9de560dSAlex Tomas if (err) 3297c9de560dSAlex Tomas goto out_err; 32980390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 3299c9de560dSAlex Tomas 3300c9de560dSAlex Tomas out_err: 330142a10addSAneesh Kumar K.V brelse(bitmap_bh); 3302c9de560dSAlex Tomas return err; 3303c9de560dSAlex Tomas } 3304c9de560dSAlex Tomas 3305c9de560dSAlex Tomas /* 3306c9de560dSAlex Tomas * here we normalize request for locality group 3307d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 3308d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 3309d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 3310b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 3311c9de560dSAlex Tomas * 3312c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 3313c9de560dSAlex Tomas */ 3314c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 3315c9de560dSAlex Tomas { 3316c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3317c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 3318c9de560dSAlex Tomas 3319c9de560dSAlex Tomas BUG_ON(lg == NULL); 3320c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 3321d3df1453SRitesh Harjani mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); 3322c9de560dSAlex Tomas } 3323c9de560dSAlex Tomas 3324c9de560dSAlex Tomas /* 3325c9de560dSAlex Tomas * Normalization means making request better in terms of 3326c9de560dSAlex Tomas * size and alignment 3327c9de560dSAlex Tomas */ 33284ddfef7bSEric Sandeen static noinline_for_stack void 33294ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 3330c9de560dSAlex Tomas struct ext4_allocation_request *ar) 3331c9de560dSAlex Tomas { 333253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3333c9de560dSAlex Tomas int bsbits, max; 3334c9de560dSAlex Tomas ext4_lblk_t end; 33351592d2c5SCurt Wohlgemuth loff_t size, start_off; 33361592d2c5SCurt Wohlgemuth loff_t orig_size __maybe_unused; 33375a0790c2SAndi Kleen ext4_lblk_t start; 3338c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 33399a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 3340c9de560dSAlex Tomas 3341c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 3342c9de560dSAlex Tomas do not need preallocation */ 3343c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3344c9de560dSAlex Tomas return; 3345c9de560dSAlex Tomas 3346c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3347c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3348c9de560dSAlex Tomas return; 3349c9de560dSAlex Tomas 3350c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3351c9de560dSAlex Tomas * required (it's a tail, for example) */ 3352c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3353c9de560dSAlex Tomas return; 3354c9de560dSAlex Tomas 3355c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3356c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3357c9de560dSAlex Tomas return ; 3358c9de560dSAlex Tomas } 3359c9de560dSAlex Tomas 3360c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3361c9de560dSAlex Tomas 3362c9de560dSAlex Tomas /* first, let's learn actual file size 3363c9de560dSAlex Tomas * given current request is allocated */ 336453accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 3365c9de560dSAlex Tomas size = size << bsbits; 3366c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3367c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 33685a0790c2SAndi Kleen orig_size = size; 3369c9de560dSAlex Tomas 33701930479cSValerie Clement /* max size of free chunks */ 33711930479cSValerie Clement max = 2 << bsbits; 3372c9de560dSAlex Tomas 33731930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 33741930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 3375c9de560dSAlex Tomas 3376c9de560dSAlex Tomas /* first, try to predict filesize */ 3377c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3378c9de560dSAlex Tomas start_off = 0; 3379c9de560dSAlex Tomas if (size <= 16 * 1024) { 3380c9de560dSAlex Tomas size = 16 * 1024; 3381c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3382c9de560dSAlex Tomas size = 32 * 1024; 3383c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3384c9de560dSAlex Tomas size = 64 * 1024; 3385c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3386c9de560dSAlex Tomas size = 128 * 1024; 3387c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3388c9de560dSAlex Tomas size = 256 * 1024; 3389c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3390c9de560dSAlex Tomas size = 512 * 1024; 3391c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3392c9de560dSAlex Tomas size = 1024 * 1024; 33931930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 3394c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 33951930479cSValerie Clement (21 - bsbits)) << 21; 33961930479cSValerie Clement size = 2 * 1024 * 1024; 33971930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 3398c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3399c9de560dSAlex Tomas (22 - bsbits)) << 22; 3400c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3401c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 34021930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 3403c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3404c9de560dSAlex Tomas (23 - bsbits)) << 23; 3405c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3406c9de560dSAlex Tomas } else { 3407c9de560dSAlex Tomas start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; 3408b27b1535SXiaoguang Wang size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), 3409b27b1535SXiaoguang Wang ac->ac_o_ex.fe_len) << bsbits; 3410c9de560dSAlex Tomas } 34115a0790c2SAndi Kleen size = size >> bsbits; 34125a0790c2SAndi Kleen start = start_off >> bsbits; 3413c9de560dSAlex Tomas 3414c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 3415c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 3416c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 3417c9de560dSAlex Tomas start = ar->lleft + 1; 3418c9de560dSAlex Tomas } 3419c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 3420c9de560dSAlex Tomas size -= start + size - ar->lright; 3421c9de560dSAlex Tomas 3422cd648b8aSJan Kara /* 3423cd648b8aSJan Kara * Trim allocation request for filesystems with artificially small 3424cd648b8aSJan Kara * groups. 3425cd648b8aSJan Kara */ 3426cd648b8aSJan Kara if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) 3427cd648b8aSJan Kara size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); 3428cd648b8aSJan Kara 3429c9de560dSAlex Tomas end = start + size; 3430c9de560dSAlex Tomas 3431c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 3432c9de560dSAlex Tomas rcu_read_lock(); 34339a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3434498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3435c9de560dSAlex Tomas 3436c9de560dSAlex Tomas if (pa->pa_deleted) 3437c9de560dSAlex Tomas continue; 3438c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3439c9de560dSAlex Tomas if (pa->pa_deleted) { 3440c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3441c9de560dSAlex Tomas continue; 3442c9de560dSAlex Tomas } 3443c9de560dSAlex Tomas 344453accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 344553accfa9STheodore Ts'o pa->pa_len); 3446c9de560dSAlex Tomas 3447c9de560dSAlex Tomas /* PA must not overlap original request */ 3448c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 3449c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 3450c9de560dSAlex Tomas 345138877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 345238877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 3453c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3454c9de560dSAlex Tomas continue; 3455c9de560dSAlex Tomas } 3456c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3457c9de560dSAlex Tomas 345838877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 3459c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3460c9de560dSAlex Tomas BUG_ON(pa_end < start); 3461c9de560dSAlex Tomas start = pa_end; 346238877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3463c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3464c9de560dSAlex Tomas end = pa->pa_lstart; 3465c9de560dSAlex Tomas } 3466c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3467c9de560dSAlex Tomas } 3468c9de560dSAlex Tomas rcu_read_unlock(); 3469c9de560dSAlex Tomas size = end - start; 3470c9de560dSAlex Tomas 3471c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3472c9de560dSAlex Tomas rcu_read_lock(); 34739a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3474498e5f24STheodore Ts'o ext4_lblk_t pa_end; 347553accfa9STheodore Ts'o 3476c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3477c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 347853accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 347953accfa9STheodore Ts'o pa->pa_len); 3480c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3481c9de560dSAlex Tomas } 3482c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3483c9de560dSAlex Tomas } 3484c9de560dSAlex Tomas rcu_read_unlock(); 3485c9de560dSAlex Tomas 3486c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3487c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 34889d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 34899d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 3490c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3491c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3492dfe076c1SDmitry Monakhov BUG(); 3493c9de560dSAlex Tomas } 3494b5b60778SMaurizio Lombardi BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 3495c9de560dSAlex Tomas 3496c9de560dSAlex Tomas /* now prepare goal request */ 3497c9de560dSAlex Tomas 3498c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3499c9de560dSAlex Tomas * placement or satisfy big request as is */ 3500c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 350153accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 3502c9de560dSAlex Tomas 3503c9de560dSAlex Tomas /* define goal start in order to merge */ 3504c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3505c9de560dSAlex Tomas /* merge to the right */ 3506c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3507c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3508c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3509c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3510c9de560dSAlex Tomas } 3511c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3512c9de560dSAlex Tomas /* merge to the left */ 3513c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3514c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3515c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3516c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3517c9de560dSAlex Tomas } 3518c9de560dSAlex Tomas 3519d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, 3520d3df1453SRitesh Harjani orig_size, start); 3521c9de560dSAlex Tomas } 3522c9de560dSAlex Tomas 3523c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3524c9de560dSAlex Tomas { 3525c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3526c9de560dSAlex Tomas 3527c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3528c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3529c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3530291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 3531c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3532c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3533c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3534c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3535c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3536c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3537c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3538c9de560dSAlex Tomas } 3539c9de560dSAlex Tomas 3540296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 3541296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 3542296c355cSTheodore Ts'o else 3543296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 3544c9de560dSAlex Tomas } 3545c9de560dSAlex Tomas 3546c9de560dSAlex Tomas /* 3547b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 3548b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 3549b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 3550b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 3551b844167eSCurt Wohlgemuth */ 3552b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 3553b844167eSCurt Wohlgemuth { 3554b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 355586f0afd4STheodore Ts'o struct ext4_buddy e4b; 355686f0afd4STheodore Ts'o int err; 3557b844167eSCurt Wohlgemuth 355886f0afd4STheodore Ts'o if (pa == NULL) { 3559c99d1e6eSTheodore Ts'o if (ac->ac_f_ex.fe_len == 0) 3560c99d1e6eSTheodore Ts'o return; 356186f0afd4STheodore Ts'o err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); 356286f0afd4STheodore Ts'o if (err) { 356386f0afd4STheodore Ts'o /* 356486f0afd4STheodore Ts'o * This should never happen since we pin the 356586f0afd4STheodore Ts'o * pages in the ext4_allocation_context so 356686f0afd4STheodore Ts'o * ext4_mb_load_buddy() should never fail. 356786f0afd4STheodore Ts'o */ 356886f0afd4STheodore Ts'o WARN(1, "mb_load_buddy failed (%d)", err); 356986f0afd4STheodore Ts'o return; 357086f0afd4STheodore Ts'o } 357186f0afd4STheodore Ts'o ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 357286f0afd4STheodore Ts'o mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, 357386f0afd4STheodore Ts'o ac->ac_f_ex.fe_len); 357486f0afd4STheodore Ts'o ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 3575c99d1e6eSTheodore Ts'o ext4_mb_unload_buddy(&e4b); 357686f0afd4STheodore Ts'o return; 357786f0afd4STheodore Ts'o } 357886f0afd4STheodore Ts'o if (pa->pa_type == MB_INODE_PA) 3579400db9d3SZheng Liu pa->pa_free += ac->ac_b_ex.fe_len; 3580b844167eSCurt Wohlgemuth } 3581b844167eSCurt Wohlgemuth 3582b844167eSCurt Wohlgemuth /* 3583c9de560dSAlex Tomas * use blocks preallocated to inode 3584c9de560dSAlex Tomas */ 3585c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3586c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3587c9de560dSAlex Tomas { 358853accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3589c9de560dSAlex Tomas ext4_fsblk_t start; 3590c9de560dSAlex Tomas ext4_fsblk_t end; 3591c9de560dSAlex Tomas int len; 3592c9de560dSAlex Tomas 3593c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3594c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 359553accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 359653accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 359753accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 3598c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3599c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3600c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3601c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3602c9de560dSAlex Tomas ac->ac_pa = pa; 3603c9de560dSAlex Tomas 3604c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 360553accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 3606c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3607c9de560dSAlex Tomas pa->pa_free -= len; 3608c9de560dSAlex Tomas 3609d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); 3610c9de560dSAlex Tomas } 3611c9de560dSAlex Tomas 3612c9de560dSAlex Tomas /* 3613c9de560dSAlex Tomas * use blocks preallocated to locality group 3614c9de560dSAlex Tomas */ 3615c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3616c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3617c9de560dSAlex Tomas { 361803cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 36196be2ded1SAneesh Kumar K.V 3620c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3621c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3622c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3623c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3624c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3625c9de560dSAlex Tomas ac->ac_pa = pa; 3626c9de560dSAlex Tomas 3627c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 362826346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3629c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 363026346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 363126346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3632c9de560dSAlex Tomas */ 3633d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", 3634d3df1453SRitesh Harjani pa->pa_lstart-len, len, pa); 3635c9de560dSAlex Tomas } 3636c9de560dSAlex Tomas 3637c9de560dSAlex Tomas /* 36385e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 36395e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 36405e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 36415e745b04SAneesh Kumar K.V * from the goal block. 36425e745b04SAneesh Kumar K.V */ 36435e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 36445e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 36455e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 36465e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 36475e745b04SAneesh Kumar K.V { 36485e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 36495e745b04SAneesh Kumar K.V 36505e745b04SAneesh Kumar K.V if (cpa == NULL) { 36515e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 36525e745b04SAneesh Kumar K.V return pa; 36535e745b04SAneesh Kumar K.V } 365479211c8eSAndrew Morton cur_distance = abs(goal_block - cpa->pa_pstart); 365579211c8eSAndrew Morton new_distance = abs(goal_block - pa->pa_pstart); 36565e745b04SAneesh Kumar K.V 36575a54b2f1SColy Li if (cur_distance <= new_distance) 36585e745b04SAneesh Kumar K.V return cpa; 36595e745b04SAneesh Kumar K.V 36605e745b04SAneesh Kumar K.V /* drop the previous reference */ 36615e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 36625e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 36635e745b04SAneesh Kumar K.V return pa; 36645e745b04SAneesh Kumar K.V } 36655e745b04SAneesh Kumar K.V 36665e745b04SAneesh Kumar K.V /* 3667c9de560dSAlex Tomas * search goal blocks in preallocated space 3668c9de560dSAlex Tomas */ 36694fca8f07SRitesh Harjani static noinline_for_stack bool 36704ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3671c9de560dSAlex Tomas { 367253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 36736be2ded1SAneesh Kumar K.V int order, i; 3674c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3675c9de560dSAlex Tomas struct ext4_locality_group *lg; 36765e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 36775e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 3678c9de560dSAlex Tomas 3679c9de560dSAlex Tomas /* only data can be preallocated */ 3680c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 36814fca8f07SRitesh Harjani return false; 3682c9de560dSAlex Tomas 3683c9de560dSAlex Tomas /* first, try per-file preallocation */ 3684c9de560dSAlex Tomas rcu_read_lock(); 36859a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3686c9de560dSAlex Tomas 3687c9de560dSAlex Tomas /* all fields in this condition don't change, 3688c9de560dSAlex Tomas * so we can skip locking for them */ 3689c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 369053accfa9STheodore Ts'o ac->ac_o_ex.fe_logical >= (pa->pa_lstart + 369153accfa9STheodore Ts'o EXT4_C2B(sbi, pa->pa_len))) 3692c9de560dSAlex Tomas continue; 3693c9de560dSAlex Tomas 3694fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 369512e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 369653accfa9STheodore Ts'o (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > 369753accfa9STheodore Ts'o EXT4_MAX_BLOCK_FILE_PHYS)) 3698fb0a387dSEric Sandeen continue; 3699fb0a387dSEric Sandeen 3700c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3701c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3702c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3703c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3704c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3705c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3706c9de560dSAlex Tomas ac->ac_criteria = 10; 3707c9de560dSAlex Tomas rcu_read_unlock(); 37084fca8f07SRitesh Harjani return true; 3709c9de560dSAlex Tomas } 3710c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3711c9de560dSAlex Tomas } 3712c9de560dSAlex Tomas rcu_read_unlock(); 3713c9de560dSAlex Tomas 3714c9de560dSAlex Tomas /* can we use group allocation? */ 3715c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 37164fca8f07SRitesh Harjani return false; 3717c9de560dSAlex Tomas 3718c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3719c9de560dSAlex Tomas lg = ac->ac_lg; 3720c9de560dSAlex Tomas if (lg == NULL) 37214fca8f07SRitesh Harjani return false; 37226be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 37236be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 37246be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 37256be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 3726c9de560dSAlex Tomas 3727bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 37285e745b04SAneesh Kumar K.V /* 37295e745b04SAneesh Kumar K.V * search for the prealloc space that is having 37305e745b04SAneesh Kumar K.V * minimal distance from the goal block. 37315e745b04SAneesh Kumar K.V */ 37326be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 3733c9de560dSAlex Tomas rcu_read_lock(); 37346be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 37356be2ded1SAneesh Kumar K.V pa_inode_list) { 3736c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 37376be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 37386be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 37395e745b04SAneesh Kumar K.V 37405e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 37415e745b04SAneesh Kumar K.V pa, cpa); 37425e745b04SAneesh Kumar K.V } 3743c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 37445e745b04SAneesh Kumar K.V } 37455e745b04SAneesh Kumar K.V rcu_read_unlock(); 37465e745b04SAneesh Kumar K.V } 37475e745b04SAneesh Kumar K.V if (cpa) { 37485e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 3749c9de560dSAlex Tomas ac->ac_criteria = 20; 37504fca8f07SRitesh Harjani return true; 3751c9de560dSAlex Tomas } 37524fca8f07SRitesh Harjani return false; 3753c9de560dSAlex Tomas } 3754c9de560dSAlex Tomas 3755c9de560dSAlex Tomas /* 37567a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 37577a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 37587a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 3759955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 37607a2fcbf7SAneesh Kumar K.V */ 37617a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 37627a2fcbf7SAneesh Kumar K.V ext4_group_t group) 37637a2fcbf7SAneesh Kumar K.V { 37647a2fcbf7SAneesh Kumar K.V struct rb_node *n; 37657a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 37667a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 37677a2fcbf7SAneesh Kumar K.V 37687a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 37697a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 37707a2fcbf7SAneesh Kumar K.V 37717a2fcbf7SAneesh Kumar K.V while (n) { 377218aadd47SBobi Jam entry = rb_entry(n, struct ext4_free_data, efd_node); 377318aadd47SBobi Jam ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); 37747a2fcbf7SAneesh Kumar K.V n = rb_next(n); 37757a2fcbf7SAneesh Kumar K.V } 37767a2fcbf7SAneesh Kumar K.V return; 37777a2fcbf7SAneesh Kumar K.V } 37787a2fcbf7SAneesh Kumar K.V 37797a2fcbf7SAneesh Kumar K.V /* 3780c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3781c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3782955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 3783c9de560dSAlex Tomas */ 3784089ceeccSEric Sandeen static noinline_for_stack 3785089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3786c9de560dSAlex Tomas ext4_group_t group) 3787c9de560dSAlex Tomas { 3788c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3789c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3790c9de560dSAlex Tomas struct list_head *cur; 3791c9de560dSAlex Tomas ext4_group_t groupnr; 3792c9de560dSAlex Tomas ext4_grpblk_t start; 3793c9de560dSAlex Tomas int preallocated = 0; 3794c9de560dSAlex Tomas int len; 3795c9de560dSAlex Tomas 3796c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3797c9de560dSAlex Tomas * so the only competing code is preallocation use. 3798c9de560dSAlex Tomas * we don't need any locking here 3799c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3800c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3801c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3802c9de560dSAlex Tomas * is dropping preallocation 3803c9de560dSAlex Tomas */ 3804c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3805c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3806c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3807c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3808c9de560dSAlex Tomas &groupnr, &start); 3809c9de560dSAlex Tomas len = pa->pa_len; 3810c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3811c9de560dSAlex Tomas if (unlikely(len == 0)) 3812c9de560dSAlex Tomas continue; 3813c9de560dSAlex Tomas BUG_ON(groupnr != group); 3814c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, start, len); 3815c9de560dSAlex Tomas preallocated += len; 3816c9de560dSAlex Tomas } 3817d3df1453SRitesh Harjani mb_debug(sb, "preallocated %d for group %u\n", preallocated, group); 3818c9de560dSAlex Tomas } 3819c9de560dSAlex Tomas 3820c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3821c9de560dSAlex Tomas { 3822c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3823c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 38244e8d2139SJunho Ryu 38254e8d2139SJunho Ryu BUG_ON(atomic_read(&pa->pa_count)); 38264e8d2139SJunho Ryu BUG_ON(pa->pa_deleted == 0); 3827c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3828c9de560dSAlex Tomas } 3829c9de560dSAlex Tomas 3830c9de560dSAlex Tomas /* 3831c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3832c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3833c9de560dSAlex Tomas */ 3834c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3835c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3836c9de560dSAlex Tomas { 3837a9df9a49STheodore Ts'o ext4_group_t grp; 3838d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 3839c9de560dSAlex Tomas 3840c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3841c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 38424e8d2139SJunho Ryu if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { 38434e8d2139SJunho Ryu spin_unlock(&pa->pa_lock); 38444e8d2139SJunho Ryu return; 38454e8d2139SJunho Ryu } 38464e8d2139SJunho Ryu 3847c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3848c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3849c9de560dSAlex Tomas return; 3850c9de560dSAlex Tomas } 3851c9de560dSAlex Tomas 3852c9de560dSAlex Tomas pa->pa_deleted = 1; 3853c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3854c9de560dSAlex Tomas 3855d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 3856cc0fb9adSAneesh Kumar K.V /* 3857cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 3858cc0fb9adSAneesh Kumar K.V * next group when pa is used up 3859cc0fb9adSAneesh Kumar K.V */ 3860cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3861d33a1976SEric Sandeen grp_blk--; 3862d33a1976SEric Sandeen 3863bd86298eSLukas Czerner grp = ext4_get_group_number(sb, grp_blk); 3864c9de560dSAlex Tomas 3865c9de560dSAlex Tomas /* 3866c9de560dSAlex Tomas * possible race: 3867c9de560dSAlex Tomas * 3868c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3869c9de560dSAlex Tomas * find block B in PA 3870c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3871c9de560dSAlex Tomas * mark B in on-disk bitmap 3872c9de560dSAlex Tomas * drop PA from group 3873c9de560dSAlex Tomas * mark all PAs in buddy 3874c9de560dSAlex Tomas * 3875c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3876c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3877c9de560dSAlex Tomas * against that pair 3878c9de560dSAlex Tomas */ 3879c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3880c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3881c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3882c9de560dSAlex Tomas 3883c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3884c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3885c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3886c9de560dSAlex Tomas 3887c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3888c9de560dSAlex Tomas } 3889c9de560dSAlex Tomas 3890c9de560dSAlex Tomas /* 3891c9de560dSAlex Tomas * creates new preallocated space for given inode 3892c9de560dSAlex Tomas */ 389353f86b17SRitesh Harjani static noinline_for_stack void 38944ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3895c9de560dSAlex Tomas { 3896c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 389753accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 3898c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3899c9de560dSAlex Tomas struct ext4_group_info *grp; 3900c9de560dSAlex Tomas struct ext4_inode_info *ei; 3901c9de560dSAlex Tomas 3902c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3903c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3904c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3905c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 390653f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 3907c9de560dSAlex Tomas 390853f86b17SRitesh Harjani pa = ac->ac_pa; 3909c9de560dSAlex Tomas 3910c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3911c9de560dSAlex Tomas int winl; 3912c9de560dSAlex Tomas int wins; 3913c9de560dSAlex Tomas int win; 3914c9de560dSAlex Tomas int offs; 3915c9de560dSAlex Tomas 3916c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3917c9de560dSAlex Tomas * so, found space must get proper lstart 3918c9de560dSAlex Tomas * to cover original request */ 3919c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3920c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3921c9de560dSAlex Tomas 3922c9de560dSAlex Tomas /* we're limited by original request in that 3923c9de560dSAlex Tomas * logical block must be covered any way 3924c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3925c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3926c9de560dSAlex Tomas 3927c9de560dSAlex Tomas /* also, we should cover whole original request */ 392853accfa9STheodore Ts'o wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); 3929c9de560dSAlex Tomas 3930c9de560dSAlex Tomas /* the smallest one defines real window */ 3931c9de560dSAlex Tomas win = min(winl, wins); 3932c9de560dSAlex Tomas 393353accfa9STheodore Ts'o offs = ac->ac_o_ex.fe_logical % 393453accfa9STheodore Ts'o EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3935c9de560dSAlex Tomas if (offs && offs < win) 3936c9de560dSAlex Tomas win = offs; 3937c9de560dSAlex Tomas 393853accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - 3939810da240SLukas Czerner EXT4_NUM_B2C(sbi, win); 3940c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3941c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3942c9de560dSAlex Tomas } 3943c9de560dSAlex Tomas 3944c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3945c9de560dSAlex Tomas * allocated blocks for history */ 3946c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3947c9de560dSAlex Tomas 3948c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3949c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3950c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3951c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3952c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3953d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3954d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3955c9de560dSAlex Tomas pa->pa_deleted = 0; 3956cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 3957c9de560dSAlex Tomas 3958d3df1453SRitesh Harjani mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 3959d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 39609bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 3961c9de560dSAlex Tomas 3962c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 396353accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 3964c9de560dSAlex Tomas 3965c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3966c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3967c9de560dSAlex Tomas 3968c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3969c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3970c9de560dSAlex Tomas 3971c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3972c9de560dSAlex Tomas 3973c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3974c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3975c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3976c9de560dSAlex Tomas } 3977c9de560dSAlex Tomas 3978c9de560dSAlex Tomas /* 3979c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3980c9de560dSAlex Tomas */ 398153f86b17SRitesh Harjani static noinline_for_stack void 39824ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3983c9de560dSAlex Tomas { 3984c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3985c9de560dSAlex Tomas struct ext4_locality_group *lg; 3986c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3987c9de560dSAlex Tomas struct ext4_group_info *grp; 3988c9de560dSAlex Tomas 3989c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3990c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3991c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3992c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 399353f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 3994c9de560dSAlex Tomas 399553f86b17SRitesh Harjani pa = ac->ac_pa; 3996c9de560dSAlex Tomas 3997c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3998c9de560dSAlex Tomas * allocated blocks for history */ 3999c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 4000c9de560dSAlex Tomas 4001c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4002c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 4003c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 4004c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 4005c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 40066be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 4007d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 4008c9de560dSAlex Tomas pa->pa_deleted = 0; 4009cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 4010c9de560dSAlex Tomas 4011d3df1453SRitesh Harjani mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 4012d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 40139bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 4014c9de560dSAlex Tomas 4015c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 4016c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 4017c9de560dSAlex Tomas 4018c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 4019c9de560dSAlex Tomas lg = ac->ac_lg; 4020c9de560dSAlex Tomas BUG_ON(lg == NULL); 4021c9de560dSAlex Tomas 4022c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 4023c9de560dSAlex Tomas pa->pa_inode = NULL; 4024c9de560dSAlex Tomas 4025c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 4026c9de560dSAlex Tomas 40276be2ded1SAneesh Kumar K.V /* 40286be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 40296be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 40306be2ded1SAneesh Kumar K.V */ 4031c9de560dSAlex Tomas } 4032c9de560dSAlex Tomas 403353f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 4034c9de560dSAlex Tomas { 4035c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 403653f86b17SRitesh Harjani ext4_mb_new_group_pa(ac); 4037c9de560dSAlex Tomas else 403853f86b17SRitesh Harjani ext4_mb_new_inode_pa(ac); 4039c9de560dSAlex Tomas } 4040c9de560dSAlex Tomas 4041c9de560dSAlex Tomas /* 4042c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 4043c9de560dSAlex Tomas * in-core bitmap and buddy. 4044c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 4045c9de560dSAlex Tomas * nobody else can find/use it. 4046c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 4047c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 4048c9de560dSAlex Tomas */ 40494ddfef7bSEric Sandeen static noinline_for_stack int 40504ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 40513e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 4052c9de560dSAlex Tomas { 4053c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4054c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4055498e5f24STheodore Ts'o unsigned int end; 4056498e5f24STheodore Ts'o unsigned int next; 4057c9de560dSAlex Tomas ext4_group_t group; 4058c9de560dSAlex Tomas ext4_grpblk_t bit; 4059ba80b101STheodore Ts'o unsigned long long grp_blk_start; 4060c9de560dSAlex Tomas int free = 0; 4061c9de560dSAlex Tomas 4062c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 4063c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 406453accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 4065c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 4066c9de560dSAlex Tomas end = bit + pa->pa_len; 4067c9de560dSAlex Tomas 4068c9de560dSAlex Tomas while (bit < end) { 4069ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 4070c9de560dSAlex Tomas if (bit >= end) 4071c9de560dSAlex Tomas break; 4072ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 4073d3df1453SRitesh Harjani mb_debug(sb, "free preallocated %u/%u in group %u\n", 40745a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 40755a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 4076c9de560dSAlex Tomas free += next - bit; 4077c9de560dSAlex Tomas 40783e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 407953accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 408053accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 4081a9c667f8SLukas Czerner next - bit); 4082c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 4083c9de560dSAlex Tomas bit = next + 1; 4084c9de560dSAlex Tomas } 4085c9de560dSAlex Tomas if (free != pa->pa_free) { 40869d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 408736bad423SRitesh Harjani "pa %p: logic %lu, phys. %lu, len %d", 4088c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 4089c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 409036bad423SRitesh Harjani pa->pa_len); 4091e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 409226346ff6SAneesh Kumar K.V free, pa->pa_free); 4093e56eb659SAneesh Kumar K.V /* 4094e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 4095e56eb659SAneesh Kumar K.V * from the bitmap and continue. 4096e56eb659SAneesh Kumar K.V */ 4097c9de560dSAlex Tomas } 4098c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 4099c9de560dSAlex Tomas 4100863c37fcSzhong jiang return 0; 4101c9de560dSAlex Tomas } 4102c9de560dSAlex Tomas 41034ddfef7bSEric Sandeen static noinline_for_stack int 41044ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 41053e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 4106c9de560dSAlex Tomas { 4107c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4108c9de560dSAlex Tomas ext4_group_t group; 4109c9de560dSAlex Tomas ext4_grpblk_t bit; 4110c9de560dSAlex Tomas 411160e07cf5SYongqiang Yang trace_ext4_mb_release_group_pa(sb, pa); 4112c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 4113c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 4114c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 4115c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 4116c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 41173e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 4118c9de560dSAlex Tomas 4119c9de560dSAlex Tomas return 0; 4120c9de560dSAlex Tomas } 4121c9de560dSAlex Tomas 4122c9de560dSAlex Tomas /* 4123c9de560dSAlex Tomas * releases all preallocations in given group 4124c9de560dSAlex Tomas * 4125c9de560dSAlex Tomas * first, we need to decide discard policy: 4126c9de560dSAlex Tomas * - when do we discard 4127c9de560dSAlex Tomas * 1) ENOSPC 4128c9de560dSAlex Tomas * - how many do we discard 4129c9de560dSAlex Tomas * 1) how many requested 4130c9de560dSAlex Tomas */ 41314ddfef7bSEric Sandeen static noinline_for_stack int 41324ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 4133c9de560dSAlex Tomas ext4_group_t group, int needed) 4134c9de560dSAlex Tomas { 4135c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 4136c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 4137c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 4138c9de560dSAlex Tomas struct list_head list; 4139c9de560dSAlex Tomas struct ext4_buddy e4b; 4140c9de560dSAlex Tomas int err; 4141c9de560dSAlex Tomas int busy = 0; 4142c9de560dSAlex Tomas int free = 0; 4143c9de560dSAlex Tomas 4144d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for group %u\n", group); 4145c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 4146bbc4ec77SRitesh Harjani goto out_dbg; 4147c9de560dSAlex Tomas 4148574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 41499008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 41509008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 415154d3adbcSTheodore Ts'o ext4_error_err(sb, -err, 415254d3adbcSTheodore Ts'o "Error %d reading block bitmap for %u", 41539008a58eSDarrick J. Wong err, group); 4154bbc4ec77SRitesh Harjani goto out_dbg; 4155c9de560dSAlex Tomas } 4156c9de560dSAlex Tomas 4157c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 4158ce89f46cSAneesh Kumar K.V if (err) { 41599651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 41609651e6b2SKonstantin Khlebnikov err, group); 4161ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 4162bbc4ec77SRitesh Harjani goto out_dbg; 4163ce89f46cSAneesh Kumar K.V } 4164c9de560dSAlex Tomas 4165c9de560dSAlex Tomas if (needed == 0) 41667137d7a4STheodore Ts'o needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 4167c9de560dSAlex Tomas 4168c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4169c9de560dSAlex Tomas repeat: 4170c9de560dSAlex Tomas ext4_lock_group(sb, group); 417107b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 4172c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 4173c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 4174c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4175c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4176c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4177c9de560dSAlex Tomas busy = 1; 4178c9de560dSAlex Tomas continue; 4179c9de560dSAlex Tomas } 4180c9de560dSAlex Tomas if (pa->pa_deleted) { 4181c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4182c9de560dSAlex Tomas continue; 4183c9de560dSAlex Tomas } 4184c9de560dSAlex Tomas 4185c9de560dSAlex Tomas /* seems this one can be freed ... */ 4186c9de560dSAlex Tomas pa->pa_deleted = 1; 4187c9de560dSAlex Tomas 4188c9de560dSAlex Tomas /* we can trust pa_free ... */ 4189c9de560dSAlex Tomas free += pa->pa_free; 4190c9de560dSAlex Tomas 4191c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4192c9de560dSAlex Tomas 4193c9de560dSAlex Tomas list_del(&pa->pa_group_list); 4194c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 4195c9de560dSAlex Tomas } 4196c9de560dSAlex Tomas 4197c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 4198c9de560dSAlex Tomas if (free < needed && busy) { 4199c9de560dSAlex Tomas busy = 0; 4200c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4201bb8b20edSLukas Czerner cond_resched(); 4202c9de560dSAlex Tomas goto repeat; 4203c9de560dSAlex Tomas } 4204c9de560dSAlex Tomas 4205c9de560dSAlex Tomas /* found anything to free? */ 4206c9de560dSAlex Tomas if (list_empty(&list)) { 4207c9de560dSAlex Tomas BUG_ON(free != 0); 4208d3df1453SRitesh Harjani mb_debug(sb, "Someone else may have freed PA for this group %u\n", 4209bbc4ec77SRitesh Harjani group); 4210c9de560dSAlex Tomas goto out; 4211c9de560dSAlex Tomas } 4212c9de560dSAlex Tomas 4213c9de560dSAlex Tomas /* now free all selected PAs */ 4214c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4215c9de560dSAlex Tomas 4216c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 4217c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 4218c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4219c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 4220c9de560dSAlex Tomas 4221cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 42223e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 4223c9de560dSAlex Tomas else 42243e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4225c9de560dSAlex Tomas 4226c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4227c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4228c9de560dSAlex Tomas } 4229c9de560dSAlex Tomas 4230c9de560dSAlex Tomas out: 4231c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4232e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4233c9de560dSAlex Tomas put_bh(bitmap_bh); 4234bbc4ec77SRitesh Harjani out_dbg: 4235d3df1453SRitesh Harjani mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n", 4236bbc4ec77SRitesh Harjani free, group, grp->bb_free); 4237c9de560dSAlex Tomas return free; 4238c9de560dSAlex Tomas } 4239c9de560dSAlex Tomas 4240c9de560dSAlex Tomas /* 4241c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 4242c9de560dSAlex Tomas * 4243c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 4244c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 4245c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 4246c9de560dSAlex Tomas * 4247c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 4248c9de560dSAlex Tomas */ 4249c2ea3fdeSTheodore Ts'o void ext4_discard_preallocations(struct inode *inode) 4250c9de560dSAlex Tomas { 4251c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 4252c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4253c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 4254c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 4255c9de560dSAlex Tomas ext4_group_t group = 0; 4256c9de560dSAlex Tomas struct list_head list; 4257c9de560dSAlex Tomas struct ext4_buddy e4b; 4258c9de560dSAlex Tomas int err; 4259c9de560dSAlex Tomas 4260c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 4261c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 4262c9de560dSAlex Tomas return; 4263c9de560dSAlex Tomas } 4264c9de560dSAlex Tomas 4265d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for inode %lu\n", 4266d3df1453SRitesh Harjani inode->i_ino); 42679bffad1eSTheodore Ts'o trace_ext4_discard_preallocations(inode); 4268c9de560dSAlex Tomas 4269c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4270c9de560dSAlex Tomas 4271c9de560dSAlex Tomas repeat: 4272c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 4273c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 4274c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 4275c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 4276c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 4277c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 4278c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4279c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4280c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 4281c9de560dSAlex Tomas * use preallocation while we're discarding it */ 4282c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4283c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 42849d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 42859d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 4286c9de560dSAlex Tomas WARN_ON(1); 4287c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4288c9de560dSAlex Tomas goto repeat; 4289c9de560dSAlex Tomas 4290c9de560dSAlex Tomas } 4291c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 4292c9de560dSAlex Tomas pa->pa_deleted = 1; 4293c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4294c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4295c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 4296c9de560dSAlex Tomas continue; 4297c9de560dSAlex Tomas } 4298c9de560dSAlex Tomas 4299c9de560dSAlex Tomas /* someone is deleting pa right now */ 4300c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4301c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4302c9de560dSAlex Tomas 4303c9de560dSAlex Tomas /* we have to wait here because pa_deleted 4304c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 4305c9de560dSAlex Tomas * the list. as we might be called from 4306c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 4307c9de560dSAlex Tomas * and concurrent thread which is unlinking 4308c9de560dSAlex Tomas * pa from inode's list may access already 4309c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 4310c9de560dSAlex Tomas 4311c9de560dSAlex Tomas /* XXX: if this happens too often, we can 4312c9de560dSAlex Tomas * add a flag to force wait only in case 4313c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 4314c9de560dSAlex Tomas * regular truncate */ 4315c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4316c9de560dSAlex Tomas goto repeat; 4317c9de560dSAlex Tomas } 4318c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4319c9de560dSAlex Tomas 4320c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4321cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 4322bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 4323c9de560dSAlex Tomas 43249651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 43259651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 4326ce89f46cSAneesh Kumar K.V if (err) { 432754d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 43289651e6b2SKonstantin Khlebnikov err, group); 4329ce89f46cSAneesh Kumar K.V continue; 4330ce89f46cSAneesh Kumar K.V } 4331c9de560dSAlex Tomas 4332574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 43339008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 43349008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 433554d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", 43369008a58eSDarrick J. Wong err, group); 4337e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4338ce89f46cSAneesh Kumar K.V continue; 4339c9de560dSAlex Tomas } 4340c9de560dSAlex Tomas 4341c9de560dSAlex Tomas ext4_lock_group(sb, group); 4342c9de560dSAlex Tomas list_del(&pa->pa_group_list); 43433e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4344c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4345c9de560dSAlex Tomas 4346e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4347c9de560dSAlex Tomas put_bh(bitmap_bh); 4348c9de560dSAlex Tomas 4349c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4350c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4351c9de560dSAlex Tomas } 4352c9de560dSAlex Tomas } 4353c9de560dSAlex Tomas 435453f86b17SRitesh Harjani static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) 435553f86b17SRitesh Harjani { 435653f86b17SRitesh Harjani struct ext4_prealloc_space *pa; 435753f86b17SRitesh Harjani 435853f86b17SRitesh Harjani BUG_ON(ext4_pspace_cachep == NULL); 435953f86b17SRitesh Harjani pa = kmem_cache_zalloc(ext4_pspace_cachep, GFP_NOFS); 436053f86b17SRitesh Harjani if (!pa) 436153f86b17SRitesh Harjani return -ENOMEM; 436253f86b17SRitesh Harjani atomic_set(&pa->pa_count, 1); 436353f86b17SRitesh Harjani ac->ac_pa = pa; 436453f86b17SRitesh Harjani return 0; 436553f86b17SRitesh Harjani } 436653f86b17SRitesh Harjani 436753f86b17SRitesh Harjani static void ext4_mb_pa_free(struct ext4_allocation_context *ac) 436853f86b17SRitesh Harjani { 436953f86b17SRitesh Harjani struct ext4_prealloc_space *pa = ac->ac_pa; 437053f86b17SRitesh Harjani 437153f86b17SRitesh Harjani BUG_ON(!pa); 437253f86b17SRitesh Harjani ac->ac_pa = NULL; 437353f86b17SRitesh Harjani WARN_ON(!atomic_dec_and_test(&pa->pa_count)); 437453f86b17SRitesh Harjani kmem_cache_free(ext4_pspace_cachep, pa); 437553f86b17SRitesh Harjani } 437653f86b17SRitesh Harjani 43776ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 4378e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 4379c9de560dSAlex Tomas { 4380e68cf40cSRitesh Harjani ext4_group_t i, ngroups; 4381c9de560dSAlex Tomas 4382d3df1453SRitesh Harjani if (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED) 4383e3570639SEric Sandeen return; 4384e3570639SEric Sandeen 43858df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 4386d3df1453SRitesh Harjani mb_debug(sb, "groups: "); 43878df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 4388c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 4389c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4390c9de560dSAlex Tomas ext4_grpblk_t start; 4391c9de560dSAlex Tomas struct list_head *cur; 4392c9de560dSAlex Tomas ext4_lock_group(sb, i); 4393c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4394c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 4395c9de560dSAlex Tomas pa_group_list); 4396c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4397c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4398c9de560dSAlex Tomas NULL, &start); 4399c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4400d3df1453SRitesh Harjani mb_debug(sb, "PA:%u:%d:%d\n", i, start, 4401d3df1453SRitesh Harjani pa->pa_len); 4402c9de560dSAlex Tomas } 440360bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 4404d3df1453SRitesh Harjani mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free, 4405d3df1453SRitesh Harjani grp->bb_fragments); 4406c9de560dSAlex Tomas } 4407c9de560dSAlex Tomas } 4408e68cf40cSRitesh Harjani 4409e68cf40cSRitesh Harjani static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4410e68cf40cSRitesh Harjani { 4411e68cf40cSRitesh Harjani struct super_block *sb = ac->ac_sb; 4412e68cf40cSRitesh Harjani 4413d3df1453SRitesh Harjani if (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED) 4414e68cf40cSRitesh Harjani return; 4415e68cf40cSRitesh Harjani 4416d3df1453SRitesh Harjani mb_debug(sb, "Can't allocate:" 4417e68cf40cSRitesh Harjani " Allocation context details:"); 4418d3df1453SRitesh Harjani mb_debug(sb, "status %u flags 0x%x", 4419e68cf40cSRitesh Harjani ac->ac_status, ac->ac_flags); 4420d3df1453SRitesh Harjani mb_debug(sb, "orig %lu/%lu/%lu@%lu, " 4421e68cf40cSRitesh Harjani "goal %lu/%lu/%lu@%lu, " 4422e68cf40cSRitesh Harjani "best %lu/%lu/%lu@%lu cr %d", 4423e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_group, 4424e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_start, 4425e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_len, 4426e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_logical, 4427e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_group, 4428e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_start, 4429e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_len, 4430e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_logical, 4431e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_group, 4432e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_start, 4433e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_len, 4434e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_logical, 4435e68cf40cSRitesh Harjani (int)ac->ac_criteria); 4436d3df1453SRitesh Harjani mb_debug(sb, "%u found", ac->ac_found); 4437e68cf40cSRitesh Harjani ext4_mb_show_pa(sb); 4438e68cf40cSRitesh Harjani } 4439c9de560dSAlex Tomas #else 4440e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 4441e68cf40cSRitesh Harjani { 4442e68cf40cSRitesh Harjani return; 4443e68cf40cSRitesh Harjani } 4444c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4445c9de560dSAlex Tomas { 4446e68cf40cSRitesh Harjani ext4_mb_show_pa(ac->ac_sb); 4447c9de560dSAlex Tomas return; 4448c9de560dSAlex Tomas } 4449c9de560dSAlex Tomas #endif 4450c9de560dSAlex Tomas 4451c9de560dSAlex Tomas /* 4452c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 4453c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 4454c9de560dSAlex Tomas * allocation which ever is larger 4455c9de560dSAlex Tomas * 4456b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 4457c9de560dSAlex Tomas */ 4458c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 4459c9de560dSAlex Tomas { 4460c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4461c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 4462c9de560dSAlex Tomas loff_t size, isize; 4463c9de560dSAlex Tomas 4464c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4465c9de560dSAlex Tomas return; 4466c9de560dSAlex Tomas 44674ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 44684ba74d00STheodore Ts'o return; 44694ba74d00STheodore Ts'o 447053accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 447150797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 447250797481STheodore Ts'o >> bsbits; 4473c9de560dSAlex Tomas 447482dd124cSNikolay Borisov if ((size == isize) && !ext4_fs_is_busy(sbi) && 447582dd124cSNikolay Borisov !inode_is_open_for_write(ac->ac_inode)) { 447650797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 447750797481STheodore Ts'o return; 447850797481STheodore Ts'o } 447950797481STheodore Ts'o 4480ebbe0277SRobin Dong if (sbi->s_mb_group_prealloc <= 0) { 4481ebbe0277SRobin Dong ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4482ebbe0277SRobin Dong return; 4483ebbe0277SRobin Dong } 4484ebbe0277SRobin Dong 4485c9de560dSAlex Tomas /* don't use group allocation for large files */ 448671780577STheodore Ts'o size = max(size, isize); 4487cc483f10STao Ma if (size > sbi->s_mb_stream_request) { 44884ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4489c9de560dSAlex Tomas return; 44904ba74d00STheodore Ts'o } 4491c9de560dSAlex Tomas 4492c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 4493c9de560dSAlex Tomas /* 4494c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 4495c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 4496c9de560dSAlex Tomas * request from multiple CPUs. 4497c9de560dSAlex Tomas */ 4498a0b6bc63SChristoph Lameter ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); 4499c9de560dSAlex Tomas 4500c9de560dSAlex Tomas /* we're going to use group allocation */ 4501c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4502c9de560dSAlex Tomas 4503c9de560dSAlex Tomas /* serialize all allocations in the group */ 4504c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4505c9de560dSAlex Tomas } 4506c9de560dSAlex Tomas 45074ddfef7bSEric Sandeen static noinline_for_stack int 45084ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4509c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4510c9de560dSAlex Tomas { 4511c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4512c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4513c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4514c9de560dSAlex Tomas ext4_group_t group; 4515498e5f24STheodore Ts'o unsigned int len; 4516498e5f24STheodore Ts'o ext4_fsblk_t goal; 4517c9de560dSAlex Tomas ext4_grpblk_t block; 4518c9de560dSAlex Tomas 4519c9de560dSAlex Tomas /* we can't allocate > group size */ 4520c9de560dSAlex Tomas len = ar->len; 4521c9de560dSAlex Tomas 4522c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 452340ae3487STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) 452440ae3487STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb); 4525c9de560dSAlex Tomas 4526c9de560dSAlex Tomas /* start searching from the goal */ 4527c9de560dSAlex Tomas goal = ar->goal; 4528c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4529c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4530c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4531c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4532c9de560dSAlex Tomas 4533c9de560dSAlex Tomas /* set up allocation goals */ 4534f5a44db5STheodore Ts'o ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); 4535c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4536c9de560dSAlex Tomas ac->ac_sb = sb; 4537c9de560dSAlex Tomas ac->ac_inode = ar->inode; 453853accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 4539c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4540c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4541c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 454253accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 4543c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4544c9de560dSAlex Tomas 45453cb77bd2Sbrookxu /* we have to define context: we'll work with a file or 4546c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4547c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4548c9de560dSAlex Tomas 4549d3df1453SRitesh Harjani mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " 4550c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4551c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4552c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4553c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4554c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 455582dd124cSNikolay Borisov inode_is_open_for_write(ar->inode) ? "" : "non-"); 4556c9de560dSAlex Tomas return 0; 4557c9de560dSAlex Tomas 4558c9de560dSAlex Tomas } 4559c9de560dSAlex Tomas 45606be2ded1SAneesh Kumar K.V static noinline_for_stack void 45616be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 45626be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 45636be2ded1SAneesh Kumar K.V int order, int total_entries) 45646be2ded1SAneesh Kumar K.V { 45656be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 45666be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 45676be2ded1SAneesh Kumar K.V struct list_head discard_list; 45686be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 45696be2ded1SAneesh Kumar K.V 4570d3df1453SRitesh Harjani mb_debug(sb, "discard locality group preallocation\n"); 45716be2ded1SAneesh Kumar K.V 45726be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 45736be2ded1SAneesh Kumar K.V 45746be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 45756be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 457692e9c58cSMadhuparna Bhowmik pa_inode_list, 457792e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 45786be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 45796be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 45806be2ded1SAneesh Kumar K.V /* 45816be2ded1SAneesh Kumar K.V * This is the pa that we just used 45826be2ded1SAneesh Kumar K.V * for block allocation. So don't 45836be2ded1SAneesh Kumar K.V * free that 45846be2ded1SAneesh Kumar K.V */ 45856be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 45866be2ded1SAneesh Kumar K.V continue; 45876be2ded1SAneesh Kumar K.V } 45886be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 45896be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 45906be2ded1SAneesh Kumar K.V continue; 45916be2ded1SAneesh Kumar K.V } 45926be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 4593cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 45946be2ded1SAneesh Kumar K.V 45956be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 45966be2ded1SAneesh Kumar K.V pa->pa_deleted = 1; 45976be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 45986be2ded1SAneesh Kumar K.V 45996be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 46006be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 46016be2ded1SAneesh Kumar K.V 46026be2ded1SAneesh Kumar K.V total_entries--; 46036be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 46046be2ded1SAneesh Kumar K.V /* 46056be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 46066be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 46076be2ded1SAneesh Kumar K.V * mak sure we don't call discard 46086be2ded1SAneesh Kumar K.V * soon for this list. 46096be2ded1SAneesh Kumar K.V */ 46106be2ded1SAneesh Kumar K.V break; 46116be2ded1SAneesh Kumar K.V } 46126be2ded1SAneesh Kumar K.V } 46136be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 46146be2ded1SAneesh Kumar K.V 46156be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 46169651e6b2SKonstantin Khlebnikov int err; 46176be2ded1SAneesh Kumar K.V 4618bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 46199651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 46209651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 46219651e6b2SKonstantin Khlebnikov if (err) { 462254d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 46239651e6b2SKonstantin Khlebnikov err, group); 46246be2ded1SAneesh Kumar K.V continue; 46256be2ded1SAneesh Kumar K.V } 46266be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 46276be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 46283e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 46296be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 46306be2ded1SAneesh Kumar K.V 4631e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 46326be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 46336be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 46346be2ded1SAneesh Kumar K.V } 46356be2ded1SAneesh Kumar K.V } 46366be2ded1SAneesh Kumar K.V 46376be2ded1SAneesh Kumar K.V /* 46386be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 46396be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 46406be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 46416be2ded1SAneesh Kumar K.V * 46426be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 46436be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 46446be2ded1SAneesh Kumar K.V */ 46456be2ded1SAneesh Kumar K.V 46466be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 46476be2ded1SAneesh Kumar K.V { 46486be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 46496be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 46506be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 46516be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 46526be2ded1SAneesh Kumar K.V 46536be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 46546be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 46556be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 46566be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 46576be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 4658f1167009SNiu Yawei spin_lock(&lg->lg_prealloc_lock); 46596be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 466092e9c58cSMadhuparna Bhowmik pa_inode_list, 466192e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 46626be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 46636be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 4664e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 46656be2ded1SAneesh Kumar K.V continue; 46666be2ded1SAneesh Kumar K.V } 46676be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 46686be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 46696be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 46706be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 46716be2ded1SAneesh Kumar K.V added = 1; 46726be2ded1SAneesh Kumar K.V /* 46736be2ded1SAneesh Kumar K.V * we want to count the total 46746be2ded1SAneesh Kumar K.V * number of entries in the list 46756be2ded1SAneesh Kumar K.V */ 46766be2ded1SAneesh Kumar K.V } 46776be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 46786be2ded1SAneesh Kumar K.V lg_prealloc_count++; 46796be2ded1SAneesh Kumar K.V } 46806be2ded1SAneesh Kumar K.V if (!added) 46816be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 46826be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 4683f1167009SNiu Yawei spin_unlock(&lg->lg_prealloc_lock); 46846be2ded1SAneesh Kumar K.V 46856be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 46866be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 46876be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 46886be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 46896be2ded1SAneesh Kumar K.V return; 46906be2ded1SAneesh Kumar K.V } 46916be2ded1SAneesh Kumar K.V return ; 46926be2ded1SAneesh Kumar K.V } 46936be2ded1SAneesh Kumar K.V 4694c9de560dSAlex Tomas /* 4695c9de560dSAlex Tomas * release all resource we used in allocation 4696c9de560dSAlex Tomas */ 4697c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4698c9de560dSAlex Tomas { 469953accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 47006be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 47016be2ded1SAneesh Kumar K.V if (pa) { 4702cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 4703c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 47046be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 470553accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 470653accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 47076be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 47086be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 47096be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 4710ba443916SAneesh Kumar K.V } 4711ba443916SAneesh Kumar K.V } 4712ba443916SAneesh Kumar K.V if (pa) { 47136be2ded1SAneesh Kumar K.V /* 47146be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 47156be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 47166be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 471744183d42SAmir Goldstein * doesn't grow big. 47186be2ded1SAneesh Kumar K.V */ 4719cc0fb9adSAneesh Kumar K.V if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) { 47206be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 47216be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 47226be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 47236be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 4724c9de560dSAlex Tomas } 47256be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 4726c9de560dSAlex Tomas } 4727c9de560dSAlex Tomas if (ac->ac_bitmap_page) 472809cbfeafSKirill A. Shutemov put_page(ac->ac_bitmap_page); 4729c9de560dSAlex Tomas if (ac->ac_buddy_page) 473009cbfeafSKirill A. Shutemov put_page(ac->ac_buddy_page); 4731c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4732c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4733c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4734c9de560dSAlex Tomas return 0; 4735c9de560dSAlex Tomas } 4736c9de560dSAlex Tomas 4737c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4738c9de560dSAlex Tomas { 47398df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 4740c9de560dSAlex Tomas int ret; 4741c9de560dSAlex Tomas int freed = 0; 4742c9de560dSAlex Tomas 47439bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 47448df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 4745c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4746c9de560dSAlex Tomas freed += ret; 4747c9de560dSAlex Tomas needed -= ret; 4748c9de560dSAlex Tomas } 4749c9de560dSAlex Tomas 4750c9de560dSAlex Tomas return freed; 4751c9de560dSAlex Tomas } 4752c9de560dSAlex Tomas 4753cf5e2ca6SRitesh Harjani static bool ext4_mb_discard_preallocations_should_retry(struct super_block *sb, 475407b5b8e1SRitesh Harjani struct ext4_allocation_context *ac, u64 *seq) 4755cf5e2ca6SRitesh Harjani { 4756cf5e2ca6SRitesh Harjani int freed; 475707b5b8e1SRitesh Harjani u64 seq_retry = 0; 475807b5b8e1SRitesh Harjani bool ret = false; 4759cf5e2ca6SRitesh Harjani 4760cf5e2ca6SRitesh Harjani freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 476107b5b8e1SRitesh Harjani if (freed) { 476207b5b8e1SRitesh Harjani ret = true; 476307b5b8e1SRitesh Harjani goto out_dbg; 476407b5b8e1SRitesh Harjani } 476507b5b8e1SRitesh Harjani seq_retry = ext4_get_discard_pa_seq_sum(); 476699377830SRitesh Harjani if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { 476799377830SRitesh Harjani ac->ac_flags |= EXT4_MB_STRICT_CHECK; 476807b5b8e1SRitesh Harjani *seq = seq_retry; 476907b5b8e1SRitesh Harjani ret = true; 477007b5b8e1SRitesh Harjani } 477107b5b8e1SRitesh Harjani 477207b5b8e1SRitesh Harjani out_dbg: 477307b5b8e1SRitesh Harjani mb_debug(sb, "freed %d, retry ? %s\n", freed, ret ? "yes" : "no"); 477407b5b8e1SRitesh Harjani return ret; 4775cf5e2ca6SRitesh Harjani } 4776cf5e2ca6SRitesh Harjani 4777c9de560dSAlex Tomas /* 4778c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4779c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4780c9de560dSAlex Tomas * to usual allocation 4781c9de560dSAlex Tomas */ 4782c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4783c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4784c9de560dSAlex Tomas { 4785256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4786c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4787c9de560dSAlex Tomas struct super_block *sb; 4788c9de560dSAlex Tomas ext4_fsblk_t block = 0; 478960e58e0fSMingming Cao unsigned int inquota = 0; 479053accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 479107b5b8e1SRitesh Harjani u64 seq; 4792c9de560dSAlex Tomas 4793b10a44c3STheodore Ts'o might_sleep(); 4794c9de560dSAlex Tomas sb = ar->inode->i_sb; 4795c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4796c9de560dSAlex Tomas 47979bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 4798ba80b101STheodore Ts'o 479945dc63e7SDmitry Monakhov /* Allow to use superuser reservation for quota file */ 480002749a4cSTahsin Erdogan if (ext4_is_quota_file(ar->inode)) 480145dc63e7SDmitry Monakhov ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; 480245dc63e7SDmitry Monakhov 4803e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { 480460e58e0fSMingming Cao /* Without delayed allocation we need to verify 480560e58e0fSMingming Cao * there is enough free blocks to do block allocation 480660e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 4807d2a17637SMingming Cao */ 480855f020dbSAllison Henderson while (ar->len && 4809e7d5f315STheodore Ts'o ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { 481055f020dbSAllison Henderson 4811030ba6bcSAneesh Kumar K.V /* let others to free the space */ 4812bb8b20edSLukas Czerner cond_resched(); 4813030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 4814030ba6bcSAneesh Kumar K.V } 4815030ba6bcSAneesh Kumar K.V if (!ar->len) { 4816bbc4ec77SRitesh Harjani ext4_mb_show_pa(sb); 481707031431SMingming Cao *errp = -ENOSPC; 481807031431SMingming Cao return 0; 481907031431SMingming Cao } 482053accfa9STheodore Ts'o reserv_clstrs = ar->len; 482155f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 482253accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 482353accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 482455f020dbSAllison Henderson } else { 482555f020dbSAllison Henderson while (ar->len && 482653accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 482753accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 482855f020dbSAllison Henderson 4829c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4830c9de560dSAlex Tomas ar->len--; 4831c9de560dSAlex Tomas } 483255f020dbSAllison Henderson } 483360e58e0fSMingming Cao inquota = ar->len; 4834c9de560dSAlex Tomas if (ar->len == 0) { 4835c9de560dSAlex Tomas *errp = -EDQUOT; 48366c7a120aSAditya Kali goto out; 4837c9de560dSAlex Tomas } 483860e58e0fSMingming Cao } 4839d2a17637SMingming Cao 484085556c9aSWei Yongjun ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); 4841833576b3STheodore Ts'o if (!ac) { 4842363d4251SShen Feng ar->len = 0; 4843256bdb49SEric Sandeen *errp = -ENOMEM; 48446c7a120aSAditya Kali goto out; 4845256bdb49SEric Sandeen } 4846256bdb49SEric Sandeen 4847256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4848c9de560dSAlex Tomas if (*errp) { 4849c9de560dSAlex Tomas ar->len = 0; 48506c7a120aSAditya Kali goto out; 4851c9de560dSAlex Tomas } 4852c9de560dSAlex Tomas 4853256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 485481198536SRitesh Harjani seq = this_cpu_read(discard_pa_seq); 4855256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4856256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4857256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 485853f86b17SRitesh Harjani 485953f86b17SRitesh Harjani *errp = ext4_mb_pa_alloc(ac); 486053f86b17SRitesh Harjani if (*errp) 486153f86b17SRitesh Harjani goto errout; 4862c9de560dSAlex Tomas repeat: 4863c9de560dSAlex Tomas /* allocate space in core */ 48646c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 486553f86b17SRitesh Harjani /* 486653f86b17SRitesh Harjani * pa allocated above is added to grp->bb_prealloc_list only 486753f86b17SRitesh Harjani * when we were able to allocate some block i.e. when 486853f86b17SRitesh Harjani * ac->ac_status == AC_STATUS_FOUND. 486953f86b17SRitesh Harjani * And error from above mean ac->ac_status != AC_STATUS_FOUND 487053f86b17SRitesh Harjani * So we have to free this pa here itself. 487153f86b17SRitesh Harjani */ 48722c00ef3eSAlexey Khoroshilov if (*errp) { 487353f86b17SRitesh Harjani ext4_mb_pa_free(ac); 48742c00ef3eSAlexey Khoroshilov ext4_discard_allocated_blocks(ac); 48752c00ef3eSAlexey Khoroshilov goto errout; 48762c00ef3eSAlexey Khoroshilov } 487753f86b17SRitesh Harjani if (ac->ac_status == AC_STATUS_FOUND && 487853f86b17SRitesh Harjani ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) 487953f86b17SRitesh Harjani ext4_mb_pa_free(ac); 4880c9de560dSAlex Tomas } 4881256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 488253accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 4883554a5cccSVegard Nossum if (*errp) { 4884b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 48856d138cedSEric Sandeen goto errout; 48866d138cedSEric Sandeen } else { 4887256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4888256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4889519deca0SAneesh Kumar K.V } 4890c9de560dSAlex Tomas } else { 489107b5b8e1SRitesh Harjani if (ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) 4892c9de560dSAlex Tomas goto repeat; 489353f86b17SRitesh Harjani /* 489453f86b17SRitesh Harjani * If block allocation fails then the pa allocated above 489553f86b17SRitesh Harjani * needs to be freed here itself. 489653f86b17SRitesh Harjani */ 489753f86b17SRitesh Harjani ext4_mb_pa_free(ac); 4898c9de560dSAlex Tomas *errp = -ENOSPC; 48996c7a120aSAditya Kali } 49006c7a120aSAditya Kali 49016d138cedSEric Sandeen errout: 49026c7a120aSAditya Kali if (*errp) { 4903256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4904c9de560dSAlex Tomas ar->len = 0; 4905256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4906c9de560dSAlex Tomas } 4907256bdb49SEric Sandeen ext4_mb_release_context(ac); 49086c7a120aSAditya Kali out: 49096c7a120aSAditya Kali if (ac) 4910363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 491160e58e0fSMingming Cao if (inquota && ar->len < inquota) 491253accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 49130087d9fbSAneesh Kumar K.V if (!ar->len) { 4914e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) 49150087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 491657042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 491753accfa9STheodore Ts'o reserv_clstrs); 49180087d9fbSAneesh Kumar K.V } 4919c9de560dSAlex Tomas 49209bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 4921ba80b101STheodore Ts'o 4922c9de560dSAlex Tomas return block; 4923c9de560dSAlex Tomas } 4924c9de560dSAlex Tomas 4925c894058dSAneesh Kumar K.V /* 4926c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 4927c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 4928c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 4929c894058dSAneesh Kumar K.V */ 4930a0154344SDaeho Jeong static void ext4_try_merge_freed_extent(struct ext4_sb_info *sbi, 4931a0154344SDaeho Jeong struct ext4_free_data *entry, 4932a0154344SDaeho Jeong struct ext4_free_data *new_entry, 4933a0154344SDaeho Jeong struct rb_root *entry_rb_root) 4934c894058dSAneesh Kumar K.V { 4935a0154344SDaeho Jeong if ((entry->efd_tid != new_entry->efd_tid) || 4936a0154344SDaeho Jeong (entry->efd_group != new_entry->efd_group)) 4937a0154344SDaeho Jeong return; 4938a0154344SDaeho Jeong if (entry->efd_start_cluster + entry->efd_count == 4939a0154344SDaeho Jeong new_entry->efd_start_cluster) { 4940a0154344SDaeho Jeong new_entry->efd_start_cluster = entry->efd_start_cluster; 4941a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 4942a0154344SDaeho Jeong } else if (new_entry->efd_start_cluster + new_entry->efd_count == 4943a0154344SDaeho Jeong entry->efd_start_cluster) { 4944a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 4945a0154344SDaeho Jeong } else 4946a0154344SDaeho Jeong return; 4947a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 4948a0154344SDaeho Jeong list_del(&entry->efd_list); 4949a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 4950a0154344SDaeho Jeong rb_erase(&entry->efd_node, entry_rb_root); 4951a0154344SDaeho Jeong kmem_cache_free(ext4_free_data_cachep, entry); 4952c894058dSAneesh Kumar K.V } 4953c894058dSAneesh Kumar K.V 49544ddfef7bSEric Sandeen static noinline_for_stack int 49554ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 49567a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 4957c9de560dSAlex Tomas { 4958e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 495984130193STheodore Ts'o ext4_grpblk_t cluster; 4960d08854f5STheodore Ts'o ext4_grpblk_t clusters = new_entry->efd_count; 49617a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 4962c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4963c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4964c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4965c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 4966c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 4967c894058dSAneesh Kumar K.V 49680390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 4969c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4970c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4971c9de560dSAlex Tomas 497218aadd47SBobi Jam new_node = &new_entry->efd_node; 497318aadd47SBobi Jam cluster = new_entry->efd_start_cluster; 4974c9de560dSAlex Tomas 4975c894058dSAneesh Kumar K.V if (!*n) { 4976c894058dSAneesh Kumar K.V /* first free block exent. We need to 4977c894058dSAneesh Kumar K.V protect buddy cache from being freed, 4978c9de560dSAlex Tomas * otherwise we'll refresh it from 4979c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4980c9de560dSAlex Tomas * blocks */ 498109cbfeafSKirill A. Shutemov get_page(e4b->bd_buddy_page); 498209cbfeafSKirill A. Shutemov get_page(e4b->bd_bitmap_page); 4983c894058dSAneesh Kumar K.V } 4984c894058dSAneesh Kumar K.V while (*n) { 4985c894058dSAneesh Kumar K.V parent = *n; 498618aadd47SBobi Jam entry = rb_entry(parent, struct ext4_free_data, efd_node); 498718aadd47SBobi Jam if (cluster < entry->efd_start_cluster) 4988c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 498918aadd47SBobi Jam else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) 4990c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 4991c894058dSAneesh Kumar K.V else { 4992e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 499384130193STheodore Ts'o ext4_group_first_block_no(sb, group) + 499484130193STheodore Ts'o EXT4_C2B(sbi, cluster), 4995e29136f8STheodore Ts'o "Block already on to-be-freed list"); 4996c894058dSAneesh Kumar K.V return 0; 4997c9de560dSAlex Tomas } 4998c9de560dSAlex Tomas } 4999c9de560dSAlex Tomas 5000c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 5001c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 5002c894058dSAneesh Kumar K.V 5003c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 5004c894058dSAneesh Kumar K.V node = rb_prev(new_node); 5005c894058dSAneesh Kumar K.V if (node) { 500618aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 5007a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 5008a0154344SDaeho Jeong &(db->bb_free_root)); 5009c9de560dSAlex Tomas } 5010c894058dSAneesh Kumar K.V 5011c894058dSAneesh Kumar K.V node = rb_next(new_node); 5012c894058dSAneesh Kumar K.V if (node) { 501318aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 5014a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 5015a0154344SDaeho Jeong &(db->bb_free_root)); 5016c894058dSAneesh Kumar K.V } 5017a0154344SDaeho Jeong 5018d08854f5STheodore Ts'o spin_lock(&sbi->s_md_lock); 5019a0154344SDaeho Jeong list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list); 5020d08854f5STheodore Ts'o sbi->s_mb_free_pending += clusters; 5021d08854f5STheodore Ts'o spin_unlock(&sbi->s_md_lock); 5022c9de560dSAlex Tomas return 0; 5023c9de560dSAlex Tomas } 5024c9de560dSAlex Tomas 502544338711STheodore Ts'o /** 502644338711STheodore Ts'o * ext4_free_blocks() -- Free given blocks and update quota 502744338711STheodore Ts'o * @handle: handle for this transaction 502844338711STheodore Ts'o * @inode: inode 5029c60990b3STheodore Ts'o * @bh: optional buffer of the block to be freed 5030c60990b3STheodore Ts'o * @block: starting physical block to be freed 5031c60990b3STheodore Ts'o * @count: number of blocks to be freed 50325def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 5033c9de560dSAlex Tomas */ 503444338711STheodore Ts'o void ext4_free_blocks(handle_t *handle, struct inode *inode, 5035e6362609STheodore Ts'o struct buffer_head *bh, ext4_fsblk_t block, 5036e6362609STheodore Ts'o unsigned long count, int flags) 5037c9de560dSAlex Tomas { 503826346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 5039c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 5040c9de560dSAlex Tomas struct ext4_group_desc *gdp; 5041498e5f24STheodore Ts'o unsigned int overflow; 5042c9de560dSAlex Tomas ext4_grpblk_t bit; 5043c9de560dSAlex Tomas struct buffer_head *gd_bh; 5044c9de560dSAlex Tomas ext4_group_t block_group; 5045c9de560dSAlex Tomas struct ext4_sb_info *sbi; 5046c9de560dSAlex Tomas struct ext4_buddy e4b; 504784130193STheodore Ts'o unsigned int count_clusters; 5048c9de560dSAlex Tomas int err = 0; 5049c9de560dSAlex Tomas int ret; 5050c9de560dSAlex Tomas 5051b10a44c3STheodore Ts'o might_sleep(); 5052e6362609STheodore Ts'o if (bh) { 5053e6362609STheodore Ts'o if (block) 5054e6362609STheodore Ts'o BUG_ON(block != bh->b_blocknr); 5055e6362609STheodore Ts'o else 5056e6362609STheodore Ts'o block = bh->b_blocknr; 5057e6362609STheodore Ts'o } 5058c9de560dSAlex Tomas 5059c9de560dSAlex Tomas sbi = EXT4_SB(sb); 50601f2acb60STheodore Ts'o if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 5061ce9f24ccSJan Kara !ext4_inode_block_valid(inode, block, count)) { 506212062dddSEric Sandeen ext4_error(sb, "Freeing blocks not in datazone - " 50630610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 5064c9de560dSAlex Tomas goto error_return; 5065c9de560dSAlex Tomas } 5066c9de560dSAlex Tomas 50670610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 5068e6362609STheodore Ts'o trace_ext4_free_blocks(inode, block, count, flags); 5069e6362609STheodore Ts'o 50709c02ac97SDaeho Jeong if (bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 50719c02ac97SDaeho Jeong BUG_ON(count > 1); 5072e6362609STheodore Ts'o 5073e6362609STheodore Ts'o ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 50749c02ac97SDaeho Jeong inode, bh, block); 5075e6362609STheodore Ts'o } 5076e6362609STheodore Ts'o 5077e6362609STheodore Ts'o /* 507884130193STheodore Ts'o * If the extent to be freed does not begin on a cluster 507984130193STheodore Ts'o * boundary, we need to deal with partial clusters at the 508084130193STheodore Ts'o * beginning and end of the extent. Normally we will free 508184130193STheodore Ts'o * blocks at the beginning or the end unless we are explicitly 508284130193STheodore Ts'o * requested to avoid doing so. 508384130193STheodore Ts'o */ 5084f5a44db5STheodore Ts'o overflow = EXT4_PBLK_COFF(sbi, block); 508584130193STheodore Ts'o if (overflow) { 508684130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { 508784130193STheodore Ts'o overflow = sbi->s_cluster_ratio - overflow; 508884130193STheodore Ts'o block += overflow; 508984130193STheodore Ts'o if (count > overflow) 509084130193STheodore Ts'o count -= overflow; 509184130193STheodore Ts'o else 509284130193STheodore Ts'o return; 509384130193STheodore Ts'o } else { 509484130193STheodore Ts'o block -= overflow; 509584130193STheodore Ts'o count += overflow; 509684130193STheodore Ts'o } 509784130193STheodore Ts'o } 5098f5a44db5STheodore Ts'o overflow = EXT4_LBLK_COFF(sbi, count); 509984130193STheodore Ts'o if (overflow) { 510084130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { 510184130193STheodore Ts'o if (count > overflow) 510284130193STheodore Ts'o count -= overflow; 510384130193STheodore Ts'o else 510484130193STheodore Ts'o return; 510584130193STheodore Ts'o } else 510684130193STheodore Ts'o count += sbi->s_cluster_ratio - overflow; 510784130193STheodore Ts'o } 510884130193STheodore Ts'o 51099c02ac97SDaeho Jeong if (!bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 51109c02ac97SDaeho Jeong int i; 5111f96c450dSDaeho Jeong int is_metadata = flags & EXT4_FREE_BLOCKS_METADATA; 51129c02ac97SDaeho Jeong 51139c02ac97SDaeho Jeong for (i = 0; i < count; i++) { 51149c02ac97SDaeho Jeong cond_resched(); 5115f96c450dSDaeho Jeong if (is_metadata) 51169c02ac97SDaeho Jeong bh = sb_find_get_block(inode->i_sb, block + i); 5117f96c450dSDaeho Jeong ext4_forget(handle, is_metadata, inode, bh, block + i); 51189c02ac97SDaeho Jeong } 51199c02ac97SDaeho Jeong } 51209c02ac97SDaeho Jeong 5121c9de560dSAlex Tomas do_more: 5122c9de560dSAlex Tomas overflow = 0; 5123c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 5124c9de560dSAlex Tomas 5125163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT( 5126163a203dSDarrick J. Wong ext4_get_group_info(sb, block_group)))) 5127163a203dSDarrick J. Wong return; 5128163a203dSDarrick J. Wong 5129c9de560dSAlex Tomas /* 5130c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 5131c9de560dSAlex Tomas * boundary. 5132c9de560dSAlex Tomas */ 513384130193STheodore Ts'o if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { 513484130193STheodore Ts'o overflow = EXT4_C2B(sbi, bit) + count - 513584130193STheodore Ts'o EXT4_BLOCKS_PER_GROUP(sb); 5136c9de560dSAlex Tomas count -= overflow; 5137c9de560dSAlex Tomas } 5138810da240SLukas Czerner count_clusters = EXT4_NUM_B2C(sbi, count); 5139574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 51409008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 51419008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 51429008a58eSDarrick J. Wong bitmap_bh = NULL; 5143c9de560dSAlex Tomas goto error_return; 5144ce89f46cSAneesh Kumar K.V } 5145c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 5146ce89f46cSAneesh Kumar K.V if (!gdp) { 5147ce89f46cSAneesh Kumar K.V err = -EIO; 5148c9de560dSAlex Tomas goto error_return; 5149ce89f46cSAneesh Kumar K.V } 5150c9de560dSAlex Tomas 5151c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 5152c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 5153c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 515449598e04SJun Piao sbi->s_itb_per_group) || 5155c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 515649598e04SJun Piao sbi->s_itb_per_group)) { 5157c9de560dSAlex Tomas 515812062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 51590610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 5160519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 5161519deca0SAneesh Kumar K.V goto error_return; 5162c9de560dSAlex Tomas } 5163c9de560dSAlex Tomas 5164c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 5165c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 5166c9de560dSAlex Tomas if (err) 5167c9de560dSAlex Tomas goto error_return; 5168c9de560dSAlex Tomas 5169c9de560dSAlex Tomas /* 5170c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 5171c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 5172c9de560dSAlex Tomas * using it 5173c9de560dSAlex Tomas */ 5174c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 5175c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 5176c9de560dSAlex Tomas if (err) 5177c9de560dSAlex Tomas goto error_return; 5178c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 5179c9de560dSAlex Tomas { 5180c9de560dSAlex Tomas int i; 518184130193STheodore Ts'o for (i = 0; i < count_clusters; i++) 5182c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 5183c9de560dSAlex Tomas } 5184c9de560dSAlex Tomas #endif 518584130193STheodore Ts'o trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); 5186c9de560dSAlex Tomas 5187adb7ef60SKonstantin Khlebnikov /* __GFP_NOFAIL: retry infinitely, ignore TIF_MEMDIE and memcg limit. */ 5188adb7ef60SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, 5189adb7ef60SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 5190920313a7SAneesh Kumar K.V if (err) 5191920313a7SAneesh Kumar K.V goto error_return; 5192e6362609STheodore Ts'o 5193f96c450dSDaeho Jeong /* 5194f96c450dSDaeho Jeong * We need to make sure we don't reuse the freed block until after the 5195f96c450dSDaeho Jeong * transaction is committed. We make an exception if the inode is to be 5196f96c450dSDaeho Jeong * written in writeback mode since writeback mode has weak data 5197f96c450dSDaeho Jeong * consistency guarantees. 5198f96c450dSDaeho Jeong */ 5199f96c450dSDaeho Jeong if (ext4_handle_valid(handle) && 5200f96c450dSDaeho Jeong ((flags & EXT4_FREE_BLOCKS_METADATA) || 5201f96c450dSDaeho Jeong !ext4_should_writeback_data(inode))) { 52027a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 52037a2fcbf7SAneesh Kumar K.V /* 52047444a072SMichal Hocko * We use __GFP_NOFAIL because ext4_free_blocks() is not allowed 52057444a072SMichal Hocko * to fail. 52067a2fcbf7SAneesh Kumar K.V */ 52077444a072SMichal Hocko new_entry = kmem_cache_alloc(ext4_free_data_cachep, 52087444a072SMichal Hocko GFP_NOFS|__GFP_NOFAIL); 520918aadd47SBobi Jam new_entry->efd_start_cluster = bit; 521018aadd47SBobi Jam new_entry->efd_group = block_group; 521118aadd47SBobi Jam new_entry->efd_count = count_clusters; 521218aadd47SBobi Jam new_entry->efd_tid = handle->h_transaction->t_tid; 5213955ce5f5SAneesh Kumar K.V 52147a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 521584130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 52167a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 5217c9de560dSAlex Tomas } else { 52187a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 52197a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 52207a2fcbf7SAneesh Kumar K.V * them with group lock_held 52217a2fcbf7SAneesh Kumar K.V */ 5222d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 5223a0154344SDaeho Jeong err = ext4_issue_discard(sb, block_group, bit, count, 5224a0154344SDaeho Jeong NULL); 5225d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 5226d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 5227d71c1ae2SLukas Czerner " group:%d block:%d count:%lu failed" 5228d71c1ae2SLukas Czerner " with %d", block_group, bit, count, 5229d71c1ae2SLukas Czerner err); 52308f9ff189SLukas Czerner } else 52318f9ff189SLukas Czerner EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); 5232d71c1ae2SLukas Czerner 5233955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 523484130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 523584130193STheodore Ts'o mb_free_blocks(inode, &e4b, bit, count_clusters); 5236c9de560dSAlex Tomas } 5237c9de560dSAlex Tomas 5238021b65bbSTheodore Ts'o ret = ext4_free_group_clusters(sb, gdp) + count_clusters; 5239021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, ret); 524079f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, gdp, bitmap_bh); 5241feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, gdp); 5242955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 5243c9de560dSAlex Tomas 5244772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 5245772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 524690ba983fSTheodore Ts'o atomic64_add(count_clusters, 52477c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 52487c990728SSuraj Jitindar Singh flex_group)->free_clusters); 5249772cb7c8SJose R. Santos } 5250772cb7c8SJose R. Santos 52519fe67149SEric Whitney /* 52529fe67149SEric Whitney * on a bigalloc file system, defer the s_freeclusters_counter 52539fe67149SEric Whitney * update to the caller (ext4_remove_space and friends) so they 52549fe67149SEric Whitney * can determine if a cluster freed here should be rereserved 52559fe67149SEric Whitney */ 52569fe67149SEric Whitney if (!(flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)) { 52577b415bf6SAditya Kali if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 52587b415bf6SAditya Kali dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); 52599fe67149SEric Whitney percpu_counter_add(&sbi->s_freeclusters_counter, 52609fe67149SEric Whitney count_clusters); 52619fe67149SEric Whitney } 52627d734532SJan Kara 52637d734532SJan Kara ext4_mb_unload_buddy(&e4b); 52647b415bf6SAditya Kali 52657a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 52667a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 52677a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 52687a2fcbf7SAneesh Kumar K.V 5269c9de560dSAlex Tomas /* And the group descriptor block */ 5270c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 52710390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 5272c9de560dSAlex Tomas if (!err) 5273c9de560dSAlex Tomas err = ret; 5274c9de560dSAlex Tomas 5275c9de560dSAlex Tomas if (overflow && !err) { 5276c9de560dSAlex Tomas block += count; 5277c9de560dSAlex Tomas count = overflow; 5278c9de560dSAlex Tomas put_bh(bitmap_bh); 5279c9de560dSAlex Tomas goto do_more; 5280c9de560dSAlex Tomas } 5281c9de560dSAlex Tomas error_return: 5282c9de560dSAlex Tomas brelse(bitmap_bh); 5283c9de560dSAlex Tomas ext4_std_error(sb, err); 5284c9de560dSAlex Tomas return; 5285c9de560dSAlex Tomas } 52867360d173SLukas Czerner 52877360d173SLukas Czerner /** 52880529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 52892846e820SAmir Goldstein * @handle: handle to this transaction 52902846e820SAmir Goldstein * @sb: super block 52914907cb7bSAnatol Pomozov * @block: start physical block to add to the block group 52922846e820SAmir Goldstein * @count: number of blocks to free 52932846e820SAmir Goldstein * 5294e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 52952846e820SAmir Goldstein */ 5296cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 52972846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 52982846e820SAmir Goldstein { 52992846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 53002846e820SAmir Goldstein struct buffer_head *gd_bh; 53012846e820SAmir Goldstein ext4_group_t block_group; 53022846e820SAmir Goldstein ext4_grpblk_t bit; 53032846e820SAmir Goldstein unsigned int i; 53042846e820SAmir Goldstein struct ext4_group_desc *desc; 53052846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 5306e73a347bSAmir Goldstein struct ext4_buddy e4b; 5307d77147ffSharshads int err = 0, ret, free_clusters_count; 5308d77147ffSharshads ext4_grpblk_t clusters_freed; 5309d77147ffSharshads ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); 5310d77147ffSharshads ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); 5311d77147ffSharshads unsigned long cluster_count = last_cluster - first_cluster + 1; 53122846e820SAmir Goldstein 53132846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 53142846e820SAmir Goldstein 53154740b830SYongqiang Yang if (count == 0) 53164740b830SYongqiang Yang return 0; 53174740b830SYongqiang Yang 53182846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 53192846e820SAmir Goldstein /* 53202846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 53212846e820SAmir Goldstein * boundary. 53222846e820SAmir Goldstein */ 5323d77147ffSharshads if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) { 5324d77147ffSharshads ext4_warning(sb, "too many blocks added to group %u", 5325cc7365dfSYongqiang Yang block_group); 5326cc7365dfSYongqiang Yang err = -EINVAL; 53272846e820SAmir Goldstein goto error_return; 5328cc7365dfSYongqiang Yang } 53292cd05cc3STheodore Ts'o 53302846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 53319008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 53329008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 53339008a58eSDarrick J. Wong bitmap_bh = NULL; 53342846e820SAmir Goldstein goto error_return; 5335cc7365dfSYongqiang Yang } 5336cc7365dfSYongqiang Yang 53372846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 5338cc7365dfSYongqiang Yang if (!desc) { 5339cc7365dfSYongqiang Yang err = -EIO; 53402846e820SAmir Goldstein goto error_return; 5341cc7365dfSYongqiang Yang } 53422846e820SAmir Goldstein 53432846e820SAmir Goldstein if (in_range(ext4_block_bitmap(sb, desc), block, count) || 53442846e820SAmir Goldstein in_range(ext4_inode_bitmap(sb, desc), block, count) || 53452846e820SAmir Goldstein in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) || 53462846e820SAmir Goldstein in_range(block + count - 1, ext4_inode_table(sb, desc), 53472846e820SAmir Goldstein sbi->s_itb_per_group)) { 53482846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 53492846e820SAmir Goldstein "Block = %llu, count = %lu", 53502846e820SAmir Goldstein block, count); 5351cc7365dfSYongqiang Yang err = -EINVAL; 53522846e820SAmir Goldstein goto error_return; 53532846e820SAmir Goldstein } 53542846e820SAmir Goldstein 53552cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 53562cd05cc3STheodore Ts'o err = ext4_journal_get_write_access(handle, bitmap_bh); 53572846e820SAmir Goldstein if (err) 53582846e820SAmir Goldstein goto error_return; 53592846e820SAmir Goldstein 53602846e820SAmir Goldstein /* 53612846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 53622846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 53632846e820SAmir Goldstein * using it 53642846e820SAmir Goldstein */ 53652846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 53662846e820SAmir Goldstein err = ext4_journal_get_write_access(handle, gd_bh); 53672846e820SAmir Goldstein if (err) 53682846e820SAmir Goldstein goto error_return; 5369e73a347bSAmir Goldstein 5370d77147ffSharshads for (i = 0, clusters_freed = 0; i < cluster_count; i++) { 53712846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 5372e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 53732846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 53742846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 53752846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 53762846e820SAmir Goldstein } else { 5377d77147ffSharshads clusters_freed++; 53782846e820SAmir Goldstein } 53792846e820SAmir Goldstein } 5380e73a347bSAmir Goldstein 5381e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 5382e73a347bSAmir Goldstein if (err) 5383e73a347bSAmir Goldstein goto error_return; 5384e73a347bSAmir Goldstein 5385e73a347bSAmir Goldstein /* 5386e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 5387e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 5388e73a347bSAmir Goldstein * them with group lock_held 5389e73a347bSAmir Goldstein */ 53902846e820SAmir Goldstein ext4_lock_group(sb, block_group); 5391d77147ffSharshads mb_clear_bits(bitmap_bh->b_data, bit, cluster_count); 5392d77147ffSharshads mb_free_blocks(NULL, &e4b, bit, cluster_count); 5393d77147ffSharshads free_clusters_count = clusters_freed + 5394d77147ffSharshads ext4_free_group_clusters(sb, desc); 5395d77147ffSharshads ext4_free_group_clusters_set(sb, desc, free_clusters_count); 539679f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, desc, bitmap_bh); 5397feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, desc); 53982846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 539957042651STheodore Ts'o percpu_counter_add(&sbi->s_freeclusters_counter, 5400d77147ffSharshads clusters_freed); 54012846e820SAmir Goldstein 54022846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 54032846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 5404d77147ffSharshads atomic64_add(clusters_freed, 54057c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 54067c990728SSuraj Jitindar Singh flex_group)->free_clusters); 54072846e820SAmir Goldstein } 5408e73a347bSAmir Goldstein 5409e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 54102846e820SAmir Goldstein 54112846e820SAmir Goldstein /* We dirtied the bitmap block */ 54122846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 54132846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 54142846e820SAmir Goldstein 54152846e820SAmir Goldstein /* And the group descriptor block */ 54162846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 54172846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 54182846e820SAmir Goldstein if (!err) 54192846e820SAmir Goldstein err = ret; 54202846e820SAmir Goldstein 54212846e820SAmir Goldstein error_return: 54222846e820SAmir Goldstein brelse(bitmap_bh); 54232846e820SAmir Goldstein ext4_std_error(sb, err); 5424cc7365dfSYongqiang Yang return err; 54252846e820SAmir Goldstein } 54262846e820SAmir Goldstein 54272846e820SAmir Goldstein /** 54287360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 54297360d173SLukas Czerner * @sb: super block for the file system 54307360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 54317360d173SLukas Czerner * @count: number of blocks to TRIM 54327360d173SLukas Czerner * @group: alloc. group we are working with 54337360d173SLukas Czerner * @e4b: ext4 buddy for the group 54347360d173SLukas Czerner * 54357360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 54367360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 54377360d173SLukas Czerner * be called with under the group lock. 54387360d173SLukas Czerner */ 5439d71c1ae2SLukas Czerner static int ext4_trim_extent(struct super_block *sb, int start, int count, 54407360d173SLukas Czerner ext4_group_t group, struct ext4_buddy *e4b) 5441e2cbd587Sjon ernst __releases(bitlock) 5442e2cbd587Sjon ernst __acquires(bitlock) 54437360d173SLukas Czerner { 54447360d173SLukas Czerner struct ext4_free_extent ex; 5445d71c1ae2SLukas Czerner int ret = 0; 54467360d173SLukas Czerner 5447b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 5448b3d4c2b1STao Ma 54497360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 54507360d173SLukas Czerner 54517360d173SLukas Czerner ex.fe_start = start; 54527360d173SLukas Czerner ex.fe_group = group; 54537360d173SLukas Czerner ex.fe_len = count; 54547360d173SLukas Czerner 54557360d173SLukas Czerner /* 54567360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 54577360d173SLukas Czerner * being trimmed. 54587360d173SLukas Czerner */ 54597360d173SLukas Czerner mb_mark_used(e4b, &ex); 54607360d173SLukas Czerner ext4_unlock_group(sb, group); 5461a0154344SDaeho Jeong ret = ext4_issue_discard(sb, group, start, count, NULL); 54627360d173SLukas Czerner ext4_lock_group(sb, group); 54637360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 5464d71c1ae2SLukas Czerner return ret; 54657360d173SLukas Czerner } 54667360d173SLukas Czerner 54677360d173SLukas Czerner /** 54687360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 54697360d173SLukas Czerner * @sb: super block for file system 547022612283STao Ma * @group: group to be trimmed 54717360d173SLukas Czerner * @start: first group block to examine 54727360d173SLukas Czerner * @max: last group block to examine 54737360d173SLukas Czerner * @minblocks: minimum extent block count 54747360d173SLukas Czerner * 54757360d173SLukas Czerner * ext4_trim_all_free walks through group's buddy bitmap searching for free 54767360d173SLukas Czerner * extents. When the free block is found, ext4_trim_extent is called to TRIM 54777360d173SLukas Czerner * the extent. 54787360d173SLukas Czerner * 54797360d173SLukas Czerner * 54807360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 54817360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 54827360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 54837360d173SLukas Czerner * the group buddy bitmap. This is done until whole group is scanned. 54847360d173SLukas Czerner */ 54850b75a840SLukas Czerner static ext4_grpblk_t 548678944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 548778944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 548878944086SLukas Czerner ext4_grpblk_t minblocks) 54897360d173SLukas Czerner { 54907360d173SLukas Czerner void *bitmap; 5491169ddc3eSTao Ma ext4_grpblk_t next, count = 0, free_count = 0; 549278944086SLukas Czerner struct ext4_buddy e4b; 5493d71c1ae2SLukas Czerner int ret = 0; 54947360d173SLukas Czerner 5495b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 5496b3d4c2b1STao Ma 549778944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 549878944086SLukas Czerner if (ret) { 54999651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 55009651e6b2SKonstantin Khlebnikov ret, group); 550178944086SLukas Czerner return ret; 550278944086SLukas Czerner } 550378944086SLukas Czerner bitmap = e4b.bd_bitmap; 550428739eeaSLukas Czerner 550528739eeaSLukas Czerner ext4_lock_group(sb, group); 55063d56b8d2STao Ma if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) && 55073d56b8d2STao Ma minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) 55083d56b8d2STao Ma goto out; 55093d56b8d2STao Ma 551078944086SLukas Czerner start = (e4b.bd_info->bb_first_free > start) ? 551178944086SLukas Czerner e4b.bd_info->bb_first_free : start; 55127360d173SLukas Czerner 5513913eed83SLukas Czerner while (start <= max) { 5514913eed83SLukas Czerner start = mb_find_next_zero_bit(bitmap, max + 1, start); 5515913eed83SLukas Czerner if (start > max) 55167360d173SLukas Czerner break; 5517913eed83SLukas Czerner next = mb_find_next_bit(bitmap, max + 1, start); 55187360d173SLukas Czerner 55197360d173SLukas Czerner if ((next - start) >= minblocks) { 5520d71c1ae2SLukas Czerner ret = ext4_trim_extent(sb, start, 552178944086SLukas Czerner next - start, group, &e4b); 5522d71c1ae2SLukas Czerner if (ret && ret != -EOPNOTSUPP) 5523d71c1ae2SLukas Czerner break; 5524d71c1ae2SLukas Czerner ret = 0; 55257360d173SLukas Czerner count += next - start; 55267360d173SLukas Czerner } 5527169ddc3eSTao Ma free_count += next - start; 55287360d173SLukas Czerner start = next + 1; 55297360d173SLukas Czerner 55307360d173SLukas Czerner if (fatal_signal_pending(current)) { 55317360d173SLukas Czerner count = -ERESTARTSYS; 55327360d173SLukas Czerner break; 55337360d173SLukas Czerner } 55347360d173SLukas Czerner 55357360d173SLukas Czerner if (need_resched()) { 55367360d173SLukas Czerner ext4_unlock_group(sb, group); 55377360d173SLukas Czerner cond_resched(); 55387360d173SLukas Czerner ext4_lock_group(sb, group); 55397360d173SLukas Czerner } 55407360d173SLukas Czerner 5541169ddc3eSTao Ma if ((e4b.bd_info->bb_free - free_count) < minblocks) 55427360d173SLukas Czerner break; 55437360d173SLukas Czerner } 55443d56b8d2STao Ma 5545d71c1ae2SLukas Czerner if (!ret) { 5546d71c1ae2SLukas Czerner ret = count; 55473d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 5548d71c1ae2SLukas Czerner } 55493d56b8d2STao Ma out: 55507360d173SLukas Czerner ext4_unlock_group(sb, group); 555178944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 55527360d173SLukas Czerner 55537360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 55547360d173SLukas Czerner count, group); 55557360d173SLukas Czerner 5556d71c1ae2SLukas Czerner return ret; 55577360d173SLukas Czerner } 55587360d173SLukas Czerner 55597360d173SLukas Czerner /** 55607360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 55617360d173SLukas Czerner * @sb: superblock for filesystem 55627360d173SLukas Czerner * @range: fstrim_range structure 55637360d173SLukas Czerner * 55647360d173SLukas Czerner * start: First Byte to trim 55657360d173SLukas Czerner * len: number of Bytes to trim from start 55667360d173SLukas Czerner * minlen: minimum extent length in Bytes 55677360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 55687360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 55697360d173SLukas Czerner * is invoked to trim all free space. 55707360d173SLukas Czerner */ 55717360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 55727360d173SLukas Czerner { 557378944086SLukas Czerner struct ext4_group_info *grp; 5574913eed83SLukas Czerner ext4_group_t group, first_group, last_group; 55757137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 5576913eed83SLukas Czerner uint64_t start, end, minlen, trimmed = 0; 55770f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 55780f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 5579913eed83SLukas Czerner ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); 55807360d173SLukas Czerner int ret = 0; 55817360d173SLukas Czerner 55827360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 5583913eed83SLukas Czerner end = start + (range->len >> sb->s_blocksize_bits) - 1; 5584aaf7d73eSLukas Czerner minlen = EXT4_NUM_B2C(EXT4_SB(sb), 5585aaf7d73eSLukas Czerner range->minlen >> sb->s_blocksize_bits); 55867360d173SLukas Czerner 55875de35e8dSLukas Czerner if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || 55885de35e8dSLukas Czerner start >= max_blks || 55895de35e8dSLukas Czerner range->len < sb->s_blocksize) 55907360d173SLukas Czerner return -EINVAL; 5591913eed83SLukas Czerner if (end >= max_blks) 5592913eed83SLukas Czerner end = max_blks - 1; 5593913eed83SLukas Czerner if (end <= first_data_blk) 559422f10457STao Ma goto out; 5595913eed83SLukas Czerner if (start < first_data_blk) 55960f0a25bfSJan Kara start = first_data_blk; 55977360d173SLukas Czerner 5598913eed83SLukas Czerner /* Determine first and last group to examine based on start and end */ 55997360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 56007137d7a4STheodore Ts'o &first_group, &first_cluster); 5601913eed83SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, 56027137d7a4STheodore Ts'o &last_group, &last_cluster); 56037360d173SLukas Czerner 5604913eed83SLukas Czerner /* end now represents the last cluster to discard in this group */ 5605913eed83SLukas Czerner end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 56067360d173SLukas Czerner 56077360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 560878944086SLukas Czerner grp = ext4_get_group_info(sb, group); 560978944086SLukas Czerner /* We only do this if the grp has never been initialized */ 561078944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 5611adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, GFP_NOFS); 561278944086SLukas Czerner if (ret) 56137360d173SLukas Czerner break; 56147360d173SLukas Czerner } 56157360d173SLukas Czerner 56160ba08517STao Ma /* 5617913eed83SLukas Czerner * For all the groups except the last one, last cluster will 5618913eed83SLukas Czerner * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to 5619913eed83SLukas Czerner * change it for the last group, note that last_cluster is 5620913eed83SLukas Czerner * already computed earlier by ext4_get_group_no_and_offset() 56210ba08517STao Ma */ 5622913eed83SLukas Czerner if (group == last_group) 5623913eed83SLukas Czerner end = last_cluster; 56247360d173SLukas Czerner 562578944086SLukas Czerner if (grp->bb_free >= minlen) { 56267137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 5627913eed83SLukas Czerner end, minlen); 56287360d173SLukas Czerner if (cnt < 0) { 56297360d173SLukas Czerner ret = cnt; 56307360d173SLukas Czerner break; 56317360d173SLukas Czerner } 56327360d173SLukas Czerner trimmed += cnt; 563321e7fd22SLukas Czerner } 5634913eed83SLukas Czerner 5635913eed83SLukas Czerner /* 5636913eed83SLukas Czerner * For every group except the first one, we are sure 5637913eed83SLukas Czerner * that the first cluster to discard will be cluster #0. 5638913eed83SLukas Czerner */ 56397137d7a4STheodore Ts'o first_cluster = 0; 56407360d173SLukas Czerner } 56417360d173SLukas Czerner 56423d56b8d2STao Ma if (!ret) 56433d56b8d2STao Ma atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); 56443d56b8d2STao Ma 564522f10457STao Ma out: 5646aaf7d73eSLukas Czerner range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; 56477360d173SLukas Czerner return ret; 56487360d173SLukas Czerner } 56490c9ec4beSDarrick J. Wong 56500c9ec4beSDarrick J. Wong /* Iterate all the free extents in the group. */ 56510c9ec4beSDarrick J. Wong int 56520c9ec4beSDarrick J. Wong ext4_mballoc_query_range( 56530c9ec4beSDarrick J. Wong struct super_block *sb, 56540c9ec4beSDarrick J. Wong ext4_group_t group, 56550c9ec4beSDarrick J. Wong ext4_grpblk_t start, 56560c9ec4beSDarrick J. Wong ext4_grpblk_t end, 56570c9ec4beSDarrick J. Wong ext4_mballoc_query_range_fn formatter, 56580c9ec4beSDarrick J. Wong void *priv) 56590c9ec4beSDarrick J. Wong { 56600c9ec4beSDarrick J. Wong void *bitmap; 56610c9ec4beSDarrick J. Wong ext4_grpblk_t next; 56620c9ec4beSDarrick J. Wong struct ext4_buddy e4b; 56630c9ec4beSDarrick J. Wong int error; 56640c9ec4beSDarrick J. Wong 56650c9ec4beSDarrick J. Wong error = ext4_mb_load_buddy(sb, group, &e4b); 56660c9ec4beSDarrick J. Wong if (error) 56670c9ec4beSDarrick J. Wong return error; 56680c9ec4beSDarrick J. Wong bitmap = e4b.bd_bitmap; 56690c9ec4beSDarrick J. Wong 56700c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 56710c9ec4beSDarrick J. Wong 56720c9ec4beSDarrick J. Wong start = (e4b.bd_info->bb_first_free > start) ? 56730c9ec4beSDarrick J. Wong e4b.bd_info->bb_first_free : start; 56740c9ec4beSDarrick J. Wong if (end >= EXT4_CLUSTERS_PER_GROUP(sb)) 56750c9ec4beSDarrick J. Wong end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 56760c9ec4beSDarrick J. Wong 56770c9ec4beSDarrick J. Wong while (start <= end) { 56780c9ec4beSDarrick J. Wong start = mb_find_next_zero_bit(bitmap, end + 1, start); 56790c9ec4beSDarrick J. Wong if (start > end) 56800c9ec4beSDarrick J. Wong break; 56810c9ec4beSDarrick J. Wong next = mb_find_next_bit(bitmap, end + 1, start); 56820c9ec4beSDarrick J. Wong 56830c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 56840c9ec4beSDarrick J. Wong error = formatter(sb, group, start, next - start, priv); 56850c9ec4beSDarrick J. Wong if (error) 56860c9ec4beSDarrick J. Wong goto out_unload; 56870c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 56880c9ec4beSDarrick J. Wong 56890c9ec4beSDarrick J. Wong start = next + 1; 56900c9ec4beSDarrick J. Wong } 56910c9ec4beSDarrick J. Wong 56920c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 56930c9ec4beSDarrick J. Wong out_unload: 56940c9ec4beSDarrick J. Wong ext4_mb_unload_buddy(&e4b); 56950c9ec4beSDarrick J. Wong 56960c9ec4beSDarrick J. Wong return error; 56970c9ec4beSDarrick J. Wong } 5698