1f5166768STheodore Ts'o // SPDX-License-Identifier: GPL-2.0 2c9de560dSAlex Tomas /* 3c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 4c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 5c9de560dSAlex Tomas */ 6c9de560dSAlex Tomas 7c9de560dSAlex Tomas 8c9de560dSAlex Tomas /* 9c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 10c9de560dSAlex Tomas */ 11c9de560dSAlex Tomas 1218aadd47SBobi Jam #include "ext4_jbd2.h" 138f6e39a7SMingming Cao #include "mballoc.h" 1428623c2fSTheodore Ts'o #include <linux/log2.h> 15a0b30c12STheodore Ts'o #include <linux/module.h> 165a0e3ad6STejun Heo #include <linux/slab.h> 171a5d5e5dSJeremy Cline #include <linux/nospec.h> 1866114cadSTejun Heo #include <linux/backing-dev.h> 199bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 209bffad1eSTheodore Ts'o 21c9de560dSAlex Tomas /* 22c9de560dSAlex Tomas * MUSTDO: 23c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 24c9de560dSAlex Tomas * - search for metadata in few groups 25c9de560dSAlex Tomas * 26c9de560dSAlex Tomas * TODO v4: 27c9de560dSAlex Tomas * - normalization should take into account whether file is still open 28c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 29c9de560dSAlex Tomas * - don't normalize tails 30c9de560dSAlex Tomas * - quota 31c9de560dSAlex Tomas * - reservation for superuser 32c9de560dSAlex Tomas * 33c9de560dSAlex Tomas * TODO v3: 34c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 35c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 36c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 37c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 38c9de560dSAlex Tomas * - error handling 39c9de560dSAlex Tomas */ 40c9de560dSAlex Tomas 41c9de560dSAlex Tomas /* 42c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 43c9de560dSAlex Tomas * near to the goal(block) value specified. 44c9de560dSAlex Tomas * 45b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 46b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 47b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 48b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 49b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 50b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 51b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 52b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 53b713a5ecSTheodore Ts'o * terms of number of blocks. 54c9de560dSAlex Tomas * 55c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 56b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 57c9de560dSAlex Tomas * 58b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 59b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 60b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 61b713a5ecSTheodore Ts'o * represented as: 62c9de560dSAlex Tomas * 63c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 64c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 6553accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 6653accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 67c9de560dSAlex Tomas * 68c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 69c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 70caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 71caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 72c9de560dSAlex Tomas * 73c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 74c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 75c9de560dSAlex Tomas * pa_free. 76c9de560dSAlex Tomas * 77c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 78c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 79caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 80c9de560dSAlex Tomas * 81c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 82c9de560dSAlex Tomas * 83c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 84c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 85c9de560dSAlex Tomas * 86c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 8725985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 88c9de560dSAlex Tomas * 89c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 90c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 91c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 92c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 93c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 94c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 95c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 96c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 97c9de560dSAlex Tomas * inode as: 98c9de560dSAlex Tomas * 99c9de560dSAlex Tomas * { page } 100c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 101c9de560dSAlex Tomas * 102c9de560dSAlex Tomas * 103c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 104ea1754a0SKirill A. Shutemov * take up 2 blocks. A page can contain blocks_per_page (PAGE_SIZE / 105c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 106c9de560dSAlex Tomas * which is blocks_per_page/2 107c9de560dSAlex Tomas * 108c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 109c9de560dSAlex Tomas * away when the filesystem is unmounted. 110c9de560dSAlex Tomas * 111c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 112c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 113c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 114c9de560dSAlex Tomas * 115c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 116c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 117c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 118c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 119c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 120c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 12127baebb8STheodore Ts'o * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is 12227baebb8STheodore Ts'o * dependent on the cluster size; for non-bigalloc file systems, it is 123c9de560dSAlex Tomas * 512 blocks. This can be tuned via 124d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 125c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 126c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 127b483bb77SRandy Dunlap * smallest multiple of the stripe value (sbi->s_stripe) which is 128d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 129c9de560dSAlex Tomas * 130196e402aSHarshad Shirwadkar * If "mb_optimize_scan" mount option is set, we maintain in memory group info 131196e402aSHarshad Shirwadkar * structures in two data structures: 132196e402aSHarshad Shirwadkar * 133196e402aSHarshad Shirwadkar * 1) Array of largest free order lists (sbi->s_mb_largest_free_orders) 134196e402aSHarshad Shirwadkar * 135196e402aSHarshad Shirwadkar * Locking: sbi->s_mb_largest_free_orders_locks(array of rw locks) 136196e402aSHarshad Shirwadkar * 137196e402aSHarshad Shirwadkar * This is an array of lists where the index in the array represents the 138196e402aSHarshad Shirwadkar * largest free order in the buddy bitmap of the participating group infos of 139196e402aSHarshad Shirwadkar * that list. So, there are exactly MB_NUM_ORDERS(sb) (which means total 140196e402aSHarshad Shirwadkar * number of buddy bitmap orders possible) number of lists. Group-infos are 141196e402aSHarshad Shirwadkar * placed in appropriate lists. 142196e402aSHarshad Shirwadkar * 143196e402aSHarshad Shirwadkar * 2) Average fragment size rb tree (sbi->s_mb_avg_fragment_size_root) 144196e402aSHarshad Shirwadkar * 145196e402aSHarshad Shirwadkar * Locking: sbi->s_mb_rb_lock (rwlock) 146196e402aSHarshad Shirwadkar * 147196e402aSHarshad Shirwadkar * This is a red black tree consisting of group infos and the tree is sorted 148196e402aSHarshad Shirwadkar * by average fragment sizes (which is calculated as ext4_group_info->bb_free 149196e402aSHarshad Shirwadkar * / ext4_group_info->bb_fragments). 150196e402aSHarshad Shirwadkar * 151196e402aSHarshad Shirwadkar * When "mb_optimize_scan" mount option is set, mballoc consults the above data 152196e402aSHarshad Shirwadkar * structures to decide the order in which groups are to be traversed for 153196e402aSHarshad Shirwadkar * fulfilling an allocation request. 154196e402aSHarshad Shirwadkar * 155196e402aSHarshad Shirwadkar * At CR = 0, we look for groups which have the largest_free_order >= the order 156196e402aSHarshad Shirwadkar * of the request. We directly look at the largest free order list in the data 157196e402aSHarshad Shirwadkar * structure (1) above where largest_free_order = order of the request. If that 158196e402aSHarshad Shirwadkar * list is empty, we look at remaining list in the increasing order of 159196e402aSHarshad Shirwadkar * largest_free_order. This allows us to perform CR = 0 lookup in O(1) time. 160196e402aSHarshad Shirwadkar * 161196e402aSHarshad Shirwadkar * At CR = 1, we only consider groups where average fragment size > request 162196e402aSHarshad Shirwadkar * size. So, we lookup a group which has average fragment size just above or 163196e402aSHarshad Shirwadkar * equal to request size using our rb tree (data structure 2) in O(log N) time. 164196e402aSHarshad Shirwadkar * 165196e402aSHarshad Shirwadkar * If "mb_optimize_scan" mount option is not set, mballoc traverses groups in 166196e402aSHarshad Shirwadkar * linear order which requires O(N) search time for each CR 0 and CR 1 phase. 167196e402aSHarshad Shirwadkar * 168d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 169c9de560dSAlex Tomas * 170b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 171b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 172b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 173196e402aSHarshad Shirwadkar * /sys/fs/ext4/<partition>/mb_linear_limit 174c9de560dSAlex Tomas * 175b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 176c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 177c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 178b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 179af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 180b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 181b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 182b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 183c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 184b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 185c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 186c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 187c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 188caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 189c9de560dSAlex Tomas * checked. 190c9de560dSAlex Tomas * 191196e402aSHarshad Shirwadkar * When "mb_optimize_scan" is turned on, as mentioned above, the groups may not 192196e402aSHarshad Shirwadkar * get traversed linearly. That may result in subsequent allocations being not 193196e402aSHarshad Shirwadkar * close to each other. And so, the underlying device may get filled up in a 194196e402aSHarshad Shirwadkar * non-linear fashion. While that may not matter on non-rotational devices, for 195196e402aSHarshad Shirwadkar * rotational devices that may result in higher seek times. "mb_linear_limit" 196196e402aSHarshad Shirwadkar * tells mballoc how many groups mballoc should search linearly before 197196e402aSHarshad Shirwadkar * performing consulting above data structures for more efficient lookups. For 198196e402aSHarshad Shirwadkar * non rotational devices, this value defaults to 0 and for rotational devices 199196e402aSHarshad Shirwadkar * this is set to MB_DEFAULT_LINEAR_LIMIT. 200196e402aSHarshad Shirwadkar * 201c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 202c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 203c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 204c9de560dSAlex Tomas * subsequent request. 205c9de560dSAlex Tomas */ 206c9de560dSAlex Tomas 207c9de560dSAlex Tomas /* 208c9de560dSAlex Tomas * mballoc operates on the following data: 209c9de560dSAlex Tomas * - on-disk bitmap 210c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 211c9de560dSAlex Tomas * - preallocation descriptors (PAs) 212c9de560dSAlex Tomas * 213c9de560dSAlex Tomas * there are two types of preallocations: 214c9de560dSAlex Tomas * - inode 215c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 216c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 217c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 218c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 219c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 220c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 221c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 222c9de560dSAlex Tomas * must discard all preallocated blocks. 223c9de560dSAlex Tomas * - locality group 224c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 225c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 226c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 227c9de560dSAlex Tomas * it's consumed from the beginning to the end. 228c9de560dSAlex Tomas * 229c9de560dSAlex Tomas * relation between them can be expressed as: 230c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 231c9de560dSAlex Tomas * 232c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 233c9de560dSAlex Tomas * - allocated blocks (persistent) 234c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 235c9de560dSAlex Tomas * 236c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 237c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 238c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 239c9de560dSAlex Tomas * 240c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 241c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 242c9de560dSAlex Tomas * 243c9de560dSAlex Tomas * all operations can be expressed as: 244c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 245c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 246c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 247c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 248c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 249c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 250c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 251c9de560dSAlex Tomas * is used in real operation because we can't know actual used 252c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 253c9de560dSAlex Tomas * 254c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 255c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 256c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 257c9de560dSAlex Tomas * the following knowledge: 258c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 259c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 260c9de560dSAlex Tomas * nobody can re-allocate that block 261c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 262c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 263c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 264c9de560dSAlex Tomas * block 265c9de560dSAlex Tomas * 266c9de560dSAlex Tomas * so, now we're building a concurrency table: 267c9de560dSAlex Tomas * - init buddy vs. 268c9de560dSAlex Tomas * - new PA 269c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 270c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 271c9de560dSAlex Tomas * - use inode PA 272c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 273c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 274c9de560dSAlex Tomas * - discard inode PA 275c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 276c9de560dSAlex Tomas * - use locality group PA 277c9de560dSAlex Tomas * again PA-=N must be serialized with init 278c9de560dSAlex Tomas * - discard locality group PA 279c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 280c9de560dSAlex Tomas * - new PA vs. 281c9de560dSAlex Tomas * - use inode PA 282c9de560dSAlex Tomas * i_data_sem serializes them 283c9de560dSAlex Tomas * - discard inode PA 284c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 285c9de560dSAlex Tomas * - use locality group PA 286c9de560dSAlex Tomas * some mutex should serialize them 287c9de560dSAlex Tomas * - discard locality group PA 288c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 289c9de560dSAlex Tomas * - use inode PA 290c9de560dSAlex Tomas * - use inode PA 291c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 292c9de560dSAlex Tomas * - discard inode PA 293c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 294c9de560dSAlex Tomas * - use locality group PA 295c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 296c9de560dSAlex Tomas * - discard locality group PA 297c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 298c9de560dSAlex Tomas * 299c9de560dSAlex Tomas * now we're ready to make few consequences: 300c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 301c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 302c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 303c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 304c9de560dSAlex Tomas * any discard or they're serialized somehow 305c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 306c9de560dSAlex Tomas * 307c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 308c9de560dSAlex Tomas * in this case, but we should care about concurrent init 309c9de560dSAlex Tomas * 310c9de560dSAlex Tomas */ 311c9de560dSAlex Tomas 312c9de560dSAlex Tomas /* 313c9de560dSAlex Tomas * Logic in few words: 314c9de560dSAlex Tomas * 315c9de560dSAlex Tomas * - allocation: 316c9de560dSAlex Tomas * load group 317c9de560dSAlex Tomas * find blocks 318c9de560dSAlex Tomas * mark bits in on-disk bitmap 319c9de560dSAlex Tomas * release group 320c9de560dSAlex Tomas * 321c9de560dSAlex Tomas * - use preallocation: 322c9de560dSAlex Tomas * find proper PA (per-inode or group) 323c9de560dSAlex Tomas * load group 324c9de560dSAlex Tomas * mark bits in on-disk bitmap 325c9de560dSAlex Tomas * release group 326c9de560dSAlex Tomas * release PA 327c9de560dSAlex Tomas * 328c9de560dSAlex Tomas * - free: 329c9de560dSAlex Tomas * load group 330c9de560dSAlex Tomas * mark bits in on-disk bitmap 331c9de560dSAlex Tomas * release group 332c9de560dSAlex Tomas * 333c9de560dSAlex Tomas * - discard preallocations in group: 334c9de560dSAlex Tomas * mark PAs deleted 335c9de560dSAlex Tomas * move them onto local list 336c9de560dSAlex Tomas * load on-disk bitmap 337c9de560dSAlex Tomas * load group 338c9de560dSAlex Tomas * remove PA from object (inode or locality group) 339c9de560dSAlex Tomas * mark free blocks in-core 340c9de560dSAlex Tomas * 341c9de560dSAlex Tomas * - discard inode's preallocations: 342c9de560dSAlex Tomas */ 343c9de560dSAlex Tomas 344c9de560dSAlex Tomas /* 345c9de560dSAlex Tomas * Locking rules 346c9de560dSAlex Tomas * 347c9de560dSAlex Tomas * Locks: 348c9de560dSAlex Tomas * - bitlock on a group (group) 349c9de560dSAlex Tomas * - object (inode/locality) (object) 350c9de560dSAlex Tomas * - per-pa lock (pa) 351196e402aSHarshad Shirwadkar * - cr0 lists lock (cr0) 352196e402aSHarshad Shirwadkar * - cr1 tree lock (cr1) 353c9de560dSAlex Tomas * 354c9de560dSAlex Tomas * Paths: 355c9de560dSAlex Tomas * - new pa 356c9de560dSAlex Tomas * object 357c9de560dSAlex Tomas * group 358c9de560dSAlex Tomas * 359c9de560dSAlex Tomas * - find and use pa: 360c9de560dSAlex Tomas * pa 361c9de560dSAlex Tomas * 362c9de560dSAlex Tomas * - release consumed pa: 363c9de560dSAlex Tomas * pa 364c9de560dSAlex Tomas * group 365c9de560dSAlex Tomas * object 366c9de560dSAlex Tomas * 367c9de560dSAlex Tomas * - generate in-core bitmap: 368c9de560dSAlex Tomas * group 369c9de560dSAlex Tomas * pa 370c9de560dSAlex Tomas * 371c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 372c9de560dSAlex Tomas * object 373c9de560dSAlex Tomas * pa 374c9de560dSAlex Tomas * group 375c9de560dSAlex Tomas * 376c9de560dSAlex Tomas * - discard all for given group: 377c9de560dSAlex Tomas * group 378c9de560dSAlex Tomas * pa 379c9de560dSAlex Tomas * group 380c9de560dSAlex Tomas * object 381c9de560dSAlex Tomas * 382196e402aSHarshad Shirwadkar * - allocation path (ext4_mb_regular_allocator) 383196e402aSHarshad Shirwadkar * group 384196e402aSHarshad Shirwadkar * cr0/cr1 385c9de560dSAlex Tomas */ 386c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 387c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 38818aadd47SBobi Jam static struct kmem_cache *ext4_free_data_cachep; 389fb1813f4SCurt Wohlgemuth 390fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 391fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 392fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 3932892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 394fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 395fb1813f4SCurt Wohlgemuth 396d6006186SEric Biggers static const char * const ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 3972892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 3982892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 3992892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 4002892c15dSEric Sandeen }; 4012892c15dSEric Sandeen 402c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 403c3a326a6SAneesh Kumar K.V ext4_group_t group); 4047a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 4057a2fcbf7SAneesh Kumar K.V ext4_group_t group); 40653f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac); 407c3a326a6SAneesh Kumar K.V 408196e402aSHarshad Shirwadkar static bool ext4_mb_good_group(struct ext4_allocation_context *ac, 409196e402aSHarshad Shirwadkar ext4_group_t group, int cr); 410196e402aSHarshad Shirwadkar 41107b5b8e1SRitesh Harjani /* 41207b5b8e1SRitesh Harjani * The algorithm using this percpu seq counter goes below: 41307b5b8e1SRitesh Harjani * 1. We sample the percpu discard_pa_seq counter before trying for block 41407b5b8e1SRitesh Harjani * allocation in ext4_mb_new_blocks(). 41507b5b8e1SRitesh Harjani * 2. We increment this percpu discard_pa_seq counter when we either allocate 41607b5b8e1SRitesh Harjani * or free these blocks i.e. while marking those blocks as used/free in 41707b5b8e1SRitesh Harjani * mb_mark_used()/mb_free_blocks(). 41807b5b8e1SRitesh Harjani * 3. We also increment this percpu seq counter when we successfully identify 41907b5b8e1SRitesh Harjani * that the bb_prealloc_list is not empty and hence proceed for discarding 42007b5b8e1SRitesh Harjani * of those PAs inside ext4_mb_discard_group_preallocations(). 42107b5b8e1SRitesh Harjani * 42207b5b8e1SRitesh Harjani * Now to make sure that the regular fast path of block allocation is not 42307b5b8e1SRitesh Harjani * affected, as a small optimization we only sample the percpu seq counter 42407b5b8e1SRitesh Harjani * on that cpu. Only when the block allocation fails and when freed blocks 42507b5b8e1SRitesh Harjani * found were 0, that is when we sample percpu seq counter for all cpus using 42607b5b8e1SRitesh Harjani * below function ext4_get_discard_pa_seq_sum(). This happens after making 42707b5b8e1SRitesh Harjani * sure that all the PAs on grp->bb_prealloc_list got freed or if it's empty. 42807b5b8e1SRitesh Harjani */ 42907b5b8e1SRitesh Harjani static DEFINE_PER_CPU(u64, discard_pa_seq); 43007b5b8e1SRitesh Harjani static inline u64 ext4_get_discard_pa_seq_sum(void) 43107b5b8e1SRitesh Harjani { 43207b5b8e1SRitesh Harjani int __cpu; 43307b5b8e1SRitesh Harjani u64 __seq = 0; 43407b5b8e1SRitesh Harjani 43507b5b8e1SRitesh Harjani for_each_possible_cpu(__cpu) 43607b5b8e1SRitesh Harjani __seq += per_cpu(discard_pa_seq, __cpu); 43707b5b8e1SRitesh Harjani return __seq; 43807b5b8e1SRitesh Harjani } 43907b5b8e1SRitesh Harjani 440ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 441ffad0a44SAneesh Kumar K.V { 442c9de560dSAlex Tomas #if BITS_PER_LONG == 64 443ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 444ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 445c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 446ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 447ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 448c9de560dSAlex Tomas #else 449c9de560dSAlex Tomas #error "how many bits you are?!" 450c9de560dSAlex Tomas #endif 451ffad0a44SAneesh Kumar K.V return addr; 452ffad0a44SAneesh Kumar K.V } 453c9de560dSAlex Tomas 454c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 455c9de560dSAlex Tomas { 456c9de560dSAlex Tomas /* 457c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 458c9de560dSAlex Tomas * needs unsigned long aligned address 459c9de560dSAlex Tomas */ 460ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 461c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 462c9de560dSAlex Tomas } 463c9de560dSAlex Tomas 464c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 465c9de560dSAlex Tomas { 466ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 467c9de560dSAlex Tomas ext4_set_bit(bit, addr); 468c9de560dSAlex Tomas } 469c9de560dSAlex Tomas 470c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 471c9de560dSAlex Tomas { 472ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 473c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 474c9de560dSAlex Tomas } 475c9de560dSAlex Tomas 476eabe0444SAndrey Sidorov static inline int mb_test_and_clear_bit(int bit, void *addr) 477eabe0444SAndrey Sidorov { 478eabe0444SAndrey Sidorov addr = mb_correct_addr_and_bit(&bit, addr); 479eabe0444SAndrey Sidorov return ext4_test_and_clear_bit(bit, addr); 480eabe0444SAndrey Sidorov } 481eabe0444SAndrey Sidorov 482ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 483ffad0a44SAneesh Kumar K.V { 484e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 485ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 486e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 487ffad0a44SAneesh Kumar K.V start += fix; 488ffad0a44SAneesh Kumar K.V 489e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 490e7dfb246SAneesh Kumar K.V if (ret > max) 491e7dfb246SAneesh Kumar K.V return max; 492e7dfb246SAneesh Kumar K.V return ret; 493ffad0a44SAneesh Kumar K.V } 494ffad0a44SAneesh Kumar K.V 495ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 496ffad0a44SAneesh Kumar K.V { 497e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 498ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 499e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 500ffad0a44SAneesh Kumar K.V start += fix; 501ffad0a44SAneesh Kumar K.V 502e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 503e7dfb246SAneesh Kumar K.V if (ret > max) 504e7dfb246SAneesh Kumar K.V return max; 505e7dfb246SAneesh Kumar K.V return ret; 506ffad0a44SAneesh Kumar K.V } 507ffad0a44SAneesh Kumar K.V 508c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 509c9de560dSAlex Tomas { 510c9de560dSAlex Tomas char *bb; 511c9de560dSAlex Tomas 512c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 513c9de560dSAlex Tomas BUG_ON(max == NULL); 514c9de560dSAlex Tomas 515c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 516c9de560dSAlex Tomas *max = 0; 517c9de560dSAlex Tomas return NULL; 518c9de560dSAlex Tomas } 519c9de560dSAlex Tomas 520c9de560dSAlex Tomas /* at order 0 we see each particular block */ 52184b775a3SColy Li if (order == 0) { 522c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 523c5e8f3f3STheodore Ts'o return e4b->bd_bitmap; 52484b775a3SColy Li } 525c9de560dSAlex Tomas 526c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 527c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 528c9de560dSAlex Tomas 529c9de560dSAlex Tomas return bb; 530c9de560dSAlex Tomas } 531c9de560dSAlex Tomas 532c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 533c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 534c9de560dSAlex Tomas int first, int count) 535c9de560dSAlex Tomas { 536c9de560dSAlex Tomas int i; 537c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 538c9de560dSAlex Tomas 539c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 540c9de560dSAlex Tomas return; 541bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 542c9de560dSAlex Tomas for (i = 0; i < count; i++) { 543c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 544c9de560dSAlex Tomas ext4_fsblk_t blocknr; 5455661bd68SAkinobu Mita 5465661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 54753accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 5485d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 549e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 550e29136f8STheodore Ts'o blocknr, 551e29136f8STheodore Ts'o "freeing block already freed " 552e29136f8STheodore Ts'o "(bit %u)", 553e29136f8STheodore Ts'o first + i); 554736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 555736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 556c9de560dSAlex Tomas } 557c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 558c9de560dSAlex Tomas } 559c9de560dSAlex Tomas } 560c9de560dSAlex Tomas 561c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 562c9de560dSAlex Tomas { 563c9de560dSAlex Tomas int i; 564c9de560dSAlex Tomas 565c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 566c9de560dSAlex Tomas return; 567bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 568c9de560dSAlex Tomas for (i = 0; i < count; i++) { 569c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 570c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 571c9de560dSAlex Tomas } 572c9de560dSAlex Tomas } 573c9de560dSAlex Tomas 574c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 575c9de560dSAlex Tomas { 576eb2b8ebbSRitesh Harjani if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 577eb2b8ebbSRitesh Harjani return; 578c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 579c9de560dSAlex Tomas unsigned char *b1, *b2; 580c9de560dSAlex Tomas int i; 581c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 582c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 583c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 584c9de560dSAlex Tomas if (b1[i] != b2[i]) { 5859d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 5869d8b9ec4STheodore Ts'o "corruption in group %u " 5874776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 5889d8b9ec4STheodore Ts'o "on disk/prealloc", 589c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 590c9de560dSAlex Tomas BUG(); 591c9de560dSAlex Tomas } 592c9de560dSAlex Tomas } 593c9de560dSAlex Tomas } 594c9de560dSAlex Tomas } 595c9de560dSAlex Tomas 596a3450215SRitesh Harjani static void mb_group_bb_bitmap_alloc(struct super_block *sb, 597a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 598a3450215SRitesh Harjani { 599a3450215SRitesh Harjani struct buffer_head *bh; 600a3450215SRitesh Harjani 601a3450215SRitesh Harjani grp->bb_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); 602eb2b8ebbSRitesh Harjani if (!grp->bb_bitmap) 603eb2b8ebbSRitesh Harjani return; 604a3450215SRitesh Harjani 605a3450215SRitesh Harjani bh = ext4_read_block_bitmap(sb, group); 606eb2b8ebbSRitesh Harjani if (IS_ERR_OR_NULL(bh)) { 607eb2b8ebbSRitesh Harjani kfree(grp->bb_bitmap); 608eb2b8ebbSRitesh Harjani grp->bb_bitmap = NULL; 609eb2b8ebbSRitesh Harjani return; 610eb2b8ebbSRitesh Harjani } 611a3450215SRitesh Harjani 612a3450215SRitesh Harjani memcpy(grp->bb_bitmap, bh->b_data, sb->s_blocksize); 613a3450215SRitesh Harjani put_bh(bh); 614a3450215SRitesh Harjani } 615a3450215SRitesh Harjani 616a3450215SRitesh Harjani static void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 617a3450215SRitesh Harjani { 618a3450215SRitesh Harjani kfree(grp->bb_bitmap); 619a3450215SRitesh Harjani } 620a3450215SRitesh Harjani 621c9de560dSAlex Tomas #else 622c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 623c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 624c9de560dSAlex Tomas { 625c9de560dSAlex Tomas return; 626c9de560dSAlex Tomas } 627c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 628c9de560dSAlex Tomas int first, int count) 629c9de560dSAlex Tomas { 630c9de560dSAlex Tomas return; 631c9de560dSAlex Tomas } 632c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 633c9de560dSAlex Tomas { 634c9de560dSAlex Tomas return; 635c9de560dSAlex Tomas } 636a3450215SRitesh Harjani 637a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_alloc(struct super_block *sb, 638a3450215SRitesh Harjani struct ext4_group_info *grp, ext4_group_t group) 639a3450215SRitesh Harjani { 640a3450215SRitesh Harjani return; 641a3450215SRitesh Harjani } 642a3450215SRitesh Harjani 643a3450215SRitesh Harjani static inline void mb_group_bb_bitmap_free(struct ext4_group_info *grp) 644a3450215SRitesh Harjani { 645a3450215SRitesh Harjani return; 646a3450215SRitesh Harjani } 647c9de560dSAlex Tomas #endif 648c9de560dSAlex Tomas 649c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 650c9de560dSAlex Tomas 651c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 652c9de560dSAlex Tomas do { \ 653c9de560dSAlex Tomas if (!(assert)) { \ 654c9de560dSAlex Tomas printk(KERN_EMERG \ 655c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 656c9de560dSAlex Tomas function, file, line, # assert); \ 657c9de560dSAlex Tomas BUG(); \ 658c9de560dSAlex Tomas } \ 659c9de560dSAlex Tomas } while (0) 660c9de560dSAlex Tomas 661c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 662c9de560dSAlex Tomas const char *function, int line) 663c9de560dSAlex Tomas { 664c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 665c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 666c9de560dSAlex Tomas int max; 667c9de560dSAlex Tomas int max2; 668c9de560dSAlex Tomas int i; 669c9de560dSAlex Tomas int j; 670c9de560dSAlex Tomas int k; 671c9de560dSAlex Tomas int count; 672c9de560dSAlex Tomas struct ext4_group_info *grp; 673c9de560dSAlex Tomas int fragments = 0; 674c9de560dSAlex Tomas int fstart; 675c9de560dSAlex Tomas struct list_head *cur; 676c9de560dSAlex Tomas void *buddy; 677c9de560dSAlex Tomas void *buddy2; 678c9de560dSAlex Tomas 679addd752cSChunguang Xu if (e4b->bd_info->bb_check_counter++ % 10) 680c9de560dSAlex Tomas return 0; 681c9de560dSAlex Tomas 682c9de560dSAlex Tomas while (order > 1) { 683c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 684c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 685c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 686c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 687c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 688c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 689c9de560dSAlex Tomas 690c9de560dSAlex Tomas count = 0; 691c9de560dSAlex Tomas for (i = 0; i < max; i++) { 692c9de560dSAlex Tomas 693c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 694c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 695c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 696c9de560dSAlex Tomas MB_CHECK_ASSERT( 697c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 698c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 699c9de560dSAlex Tomas MB_CHECK_ASSERT( 700c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 701c9de560dSAlex Tomas } 702c9de560dSAlex Tomas continue; 703c9de560dSAlex Tomas } 704c9de560dSAlex Tomas 7050a10da73SRobin Dong /* both bits in buddy2 must be 1 */ 706c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 707c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 708c9de560dSAlex Tomas 709c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 710c9de560dSAlex Tomas k = (i * (1 << order)) + j; 711c9de560dSAlex Tomas MB_CHECK_ASSERT( 712c5e8f3f3STheodore Ts'o !mb_test_bit(k, e4b->bd_bitmap)); 713c9de560dSAlex Tomas } 714c9de560dSAlex Tomas count++; 715c9de560dSAlex Tomas } 716c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 717c9de560dSAlex Tomas order--; 718c9de560dSAlex Tomas } 719c9de560dSAlex Tomas 720c9de560dSAlex Tomas fstart = -1; 721c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 722c9de560dSAlex Tomas for (i = 0; i < max; i++) { 723c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 724c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 725c9de560dSAlex Tomas if (fstart == -1) { 726c9de560dSAlex Tomas fragments++; 727c9de560dSAlex Tomas fstart = i; 728c9de560dSAlex Tomas } 729c9de560dSAlex Tomas continue; 730c9de560dSAlex Tomas } 731c9de560dSAlex Tomas fstart = -1; 732c9de560dSAlex Tomas /* check used bits only */ 733c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 734c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 735c9de560dSAlex Tomas k = i >> j; 736c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 737c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 738c9de560dSAlex Tomas } 739c9de560dSAlex Tomas } 740c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 741c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 742c9de560dSAlex Tomas 743c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 744c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 745c9de560dSAlex Tomas ext4_group_t groupnr; 746c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 74760bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 74860bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 749c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 75060bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 751c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 752c9de560dSAlex Tomas } 753c9de560dSAlex Tomas return 0; 754c9de560dSAlex Tomas } 755c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 756c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 75746e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 758c9de560dSAlex Tomas #else 759c9de560dSAlex Tomas #define mb_check_buddy(e4b) 760c9de560dSAlex Tomas #endif 761c9de560dSAlex Tomas 7627c786059SColy Li /* 7637c786059SColy Li * Divide blocks started from @first with length @len into 7647c786059SColy Li * smaller chunks with power of 2 blocks. 7657c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 7667c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 7677c786059SColy Li */ 768c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 769a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 770c9de560dSAlex Tomas struct ext4_group_info *grp) 771c9de560dSAlex Tomas { 772c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 773a36b4498SEric Sandeen ext4_grpblk_t min; 774a36b4498SEric Sandeen ext4_grpblk_t max; 775a36b4498SEric Sandeen ext4_grpblk_t chunk; 77669e43e8cSChandan Rajendra unsigned int border; 777c9de560dSAlex Tomas 7787137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 779c9de560dSAlex Tomas 780c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 781c9de560dSAlex Tomas 782c9de560dSAlex Tomas while (len > 0) { 783c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 784c9de560dSAlex Tomas max = ffs(first | border) - 1; 785c9de560dSAlex Tomas 786c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 787c9de560dSAlex Tomas min = fls(len) - 1; 788c9de560dSAlex Tomas 789c9de560dSAlex Tomas if (max < min) 790c9de560dSAlex Tomas min = max; 791c9de560dSAlex Tomas chunk = 1 << min; 792c9de560dSAlex Tomas 793c9de560dSAlex Tomas /* mark multiblock chunks only */ 794c9de560dSAlex Tomas grp->bb_counters[min]++; 795c9de560dSAlex Tomas if (min > 0) 796c9de560dSAlex Tomas mb_clear_bit(first >> min, 797c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 798c9de560dSAlex Tomas 799c9de560dSAlex Tomas len -= chunk; 800c9de560dSAlex Tomas first += chunk; 801c9de560dSAlex Tomas } 802c9de560dSAlex Tomas } 803c9de560dSAlex Tomas 804196e402aSHarshad Shirwadkar static void ext4_mb_rb_insert(struct rb_root *root, struct rb_node *new, 805196e402aSHarshad Shirwadkar int (*cmp)(struct rb_node *, struct rb_node *)) 806196e402aSHarshad Shirwadkar { 807196e402aSHarshad Shirwadkar struct rb_node **iter = &root->rb_node, *parent = NULL; 808196e402aSHarshad Shirwadkar 809196e402aSHarshad Shirwadkar while (*iter) { 810196e402aSHarshad Shirwadkar parent = *iter; 811196e402aSHarshad Shirwadkar if (cmp(new, *iter) > 0) 812196e402aSHarshad Shirwadkar iter = &((*iter)->rb_left); 813196e402aSHarshad Shirwadkar else 814196e402aSHarshad Shirwadkar iter = &((*iter)->rb_right); 815196e402aSHarshad Shirwadkar } 816196e402aSHarshad Shirwadkar 817196e402aSHarshad Shirwadkar rb_link_node(new, parent, iter); 818196e402aSHarshad Shirwadkar rb_insert_color(new, root); 819196e402aSHarshad Shirwadkar } 820196e402aSHarshad Shirwadkar 821196e402aSHarshad Shirwadkar static int 822196e402aSHarshad Shirwadkar ext4_mb_avg_fragment_size_cmp(struct rb_node *rb1, struct rb_node *rb2) 823196e402aSHarshad Shirwadkar { 824196e402aSHarshad Shirwadkar struct ext4_group_info *grp1 = rb_entry(rb1, 825196e402aSHarshad Shirwadkar struct ext4_group_info, 826196e402aSHarshad Shirwadkar bb_avg_fragment_size_rb); 827196e402aSHarshad Shirwadkar struct ext4_group_info *grp2 = rb_entry(rb2, 828196e402aSHarshad Shirwadkar struct ext4_group_info, 829196e402aSHarshad Shirwadkar bb_avg_fragment_size_rb); 830196e402aSHarshad Shirwadkar int num_frags_1, num_frags_2; 831196e402aSHarshad Shirwadkar 832196e402aSHarshad Shirwadkar num_frags_1 = grp1->bb_fragments ? 833196e402aSHarshad Shirwadkar grp1->bb_free / grp1->bb_fragments : 0; 834196e402aSHarshad Shirwadkar num_frags_2 = grp2->bb_fragments ? 835196e402aSHarshad Shirwadkar grp2->bb_free / grp2->bb_fragments : 0; 836196e402aSHarshad Shirwadkar 837196e402aSHarshad Shirwadkar return (num_frags_2 - num_frags_1); 838196e402aSHarshad Shirwadkar } 839196e402aSHarshad Shirwadkar 840196e402aSHarshad Shirwadkar /* 841196e402aSHarshad Shirwadkar * Reinsert grpinfo into the avg_fragment_size tree with new average 842196e402aSHarshad Shirwadkar * fragment size. 843196e402aSHarshad Shirwadkar */ 844196e402aSHarshad Shirwadkar static void 845196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(struct super_block *sb, struct ext4_group_info *grp) 846196e402aSHarshad Shirwadkar { 847196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 848196e402aSHarshad Shirwadkar 849196e402aSHarshad Shirwadkar if (!test_opt2(sb, MB_OPTIMIZE_SCAN) || grp->bb_free == 0) 850196e402aSHarshad Shirwadkar return; 851196e402aSHarshad Shirwadkar 852196e402aSHarshad Shirwadkar write_lock(&sbi->s_mb_rb_lock); 853196e402aSHarshad Shirwadkar if (!RB_EMPTY_NODE(&grp->bb_avg_fragment_size_rb)) { 854196e402aSHarshad Shirwadkar rb_erase(&grp->bb_avg_fragment_size_rb, 855196e402aSHarshad Shirwadkar &sbi->s_mb_avg_fragment_size_root); 856196e402aSHarshad Shirwadkar RB_CLEAR_NODE(&grp->bb_avg_fragment_size_rb); 857196e402aSHarshad Shirwadkar } 858196e402aSHarshad Shirwadkar 859196e402aSHarshad Shirwadkar ext4_mb_rb_insert(&sbi->s_mb_avg_fragment_size_root, 860196e402aSHarshad Shirwadkar &grp->bb_avg_fragment_size_rb, 861196e402aSHarshad Shirwadkar ext4_mb_avg_fragment_size_cmp); 862196e402aSHarshad Shirwadkar write_unlock(&sbi->s_mb_rb_lock); 863196e402aSHarshad Shirwadkar } 864196e402aSHarshad Shirwadkar 865196e402aSHarshad Shirwadkar /* 866196e402aSHarshad Shirwadkar * Choose next group by traversing largest_free_order lists. Updates *new_cr if 867196e402aSHarshad Shirwadkar * cr level needs an update. 868196e402aSHarshad Shirwadkar */ 869196e402aSHarshad Shirwadkar static void ext4_mb_choose_next_group_cr0(struct ext4_allocation_context *ac, 870196e402aSHarshad Shirwadkar int *new_cr, ext4_group_t *group, ext4_group_t ngroups) 871196e402aSHarshad Shirwadkar { 872196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 873196e402aSHarshad Shirwadkar struct ext4_group_info *iter, *grp; 874196e402aSHarshad Shirwadkar int i; 875196e402aSHarshad Shirwadkar 876196e402aSHarshad Shirwadkar if (ac->ac_status == AC_STATUS_FOUND) 877196e402aSHarshad Shirwadkar return; 878196e402aSHarshad Shirwadkar 879196e402aSHarshad Shirwadkar if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR0_OPTIMIZED)) 880196e402aSHarshad Shirwadkar atomic_inc(&sbi->s_bal_cr0_bad_suggestions); 881196e402aSHarshad Shirwadkar 882196e402aSHarshad Shirwadkar grp = NULL; 883196e402aSHarshad Shirwadkar for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) { 884196e402aSHarshad Shirwadkar if (list_empty(&sbi->s_mb_largest_free_orders[i])) 885196e402aSHarshad Shirwadkar continue; 886196e402aSHarshad Shirwadkar read_lock(&sbi->s_mb_largest_free_orders_locks[i]); 887196e402aSHarshad Shirwadkar if (list_empty(&sbi->s_mb_largest_free_orders[i])) { 888196e402aSHarshad Shirwadkar read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); 889196e402aSHarshad Shirwadkar continue; 890196e402aSHarshad Shirwadkar } 891196e402aSHarshad Shirwadkar grp = NULL; 892196e402aSHarshad Shirwadkar list_for_each_entry(iter, &sbi->s_mb_largest_free_orders[i], 893196e402aSHarshad Shirwadkar bb_largest_free_order_node) { 894196e402aSHarshad Shirwadkar if (sbi->s_mb_stats) 895196e402aSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_groups_considered[0]); 896196e402aSHarshad Shirwadkar if (likely(ext4_mb_good_group(ac, iter->bb_group, 0))) { 897196e402aSHarshad Shirwadkar grp = iter; 898196e402aSHarshad Shirwadkar break; 899196e402aSHarshad Shirwadkar } 900196e402aSHarshad Shirwadkar } 901196e402aSHarshad Shirwadkar read_unlock(&sbi->s_mb_largest_free_orders_locks[i]); 902196e402aSHarshad Shirwadkar if (grp) 903196e402aSHarshad Shirwadkar break; 904196e402aSHarshad Shirwadkar } 905196e402aSHarshad Shirwadkar 906196e402aSHarshad Shirwadkar if (!grp) { 907196e402aSHarshad Shirwadkar /* Increment cr and search again */ 908196e402aSHarshad Shirwadkar *new_cr = 1; 909196e402aSHarshad Shirwadkar } else { 910196e402aSHarshad Shirwadkar *group = grp->bb_group; 911196e402aSHarshad Shirwadkar ac->ac_last_optimal_group = *group; 912196e402aSHarshad Shirwadkar ac->ac_flags |= EXT4_MB_CR0_OPTIMIZED; 913196e402aSHarshad Shirwadkar } 914196e402aSHarshad Shirwadkar } 915196e402aSHarshad Shirwadkar 916196e402aSHarshad Shirwadkar /* 917196e402aSHarshad Shirwadkar * Choose next group by traversing average fragment size tree. Updates *new_cr 918196e402aSHarshad Shirwadkar * if cr lvel needs an update. Sets EXT4_MB_SEARCH_NEXT_LINEAR to indicate that 919196e402aSHarshad Shirwadkar * the linear search should continue for one iteration since there's lock 920196e402aSHarshad Shirwadkar * contention on the rb tree lock. 921196e402aSHarshad Shirwadkar */ 922196e402aSHarshad Shirwadkar static void ext4_mb_choose_next_group_cr1(struct ext4_allocation_context *ac, 923196e402aSHarshad Shirwadkar int *new_cr, ext4_group_t *group, ext4_group_t ngroups) 924196e402aSHarshad Shirwadkar { 925196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 926196e402aSHarshad Shirwadkar int avg_fragment_size, best_so_far; 927196e402aSHarshad Shirwadkar struct rb_node *node, *found; 928196e402aSHarshad Shirwadkar struct ext4_group_info *grp; 929196e402aSHarshad Shirwadkar 930196e402aSHarshad Shirwadkar /* 931196e402aSHarshad Shirwadkar * If there is contention on the lock, instead of waiting for the lock 932196e402aSHarshad Shirwadkar * to become available, just continue searching lineraly. We'll resume 933196e402aSHarshad Shirwadkar * our rb tree search later starting at ac->ac_last_optimal_group. 934196e402aSHarshad Shirwadkar */ 935196e402aSHarshad Shirwadkar if (!read_trylock(&sbi->s_mb_rb_lock)) { 936196e402aSHarshad Shirwadkar ac->ac_flags |= EXT4_MB_SEARCH_NEXT_LINEAR; 937196e402aSHarshad Shirwadkar return; 938196e402aSHarshad Shirwadkar } 939196e402aSHarshad Shirwadkar 940196e402aSHarshad Shirwadkar if (unlikely(ac->ac_flags & EXT4_MB_CR1_OPTIMIZED)) { 941196e402aSHarshad Shirwadkar if (sbi->s_mb_stats) 942196e402aSHarshad Shirwadkar atomic_inc(&sbi->s_bal_cr1_bad_suggestions); 943196e402aSHarshad Shirwadkar /* We have found something at CR 1 in the past */ 944196e402aSHarshad Shirwadkar grp = ext4_get_group_info(ac->ac_sb, ac->ac_last_optimal_group); 945196e402aSHarshad Shirwadkar for (found = rb_next(&grp->bb_avg_fragment_size_rb); found != NULL; 946196e402aSHarshad Shirwadkar found = rb_next(found)) { 947196e402aSHarshad Shirwadkar grp = rb_entry(found, struct ext4_group_info, 948196e402aSHarshad Shirwadkar bb_avg_fragment_size_rb); 949196e402aSHarshad Shirwadkar if (sbi->s_mb_stats) 950196e402aSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_groups_considered[1]); 951196e402aSHarshad Shirwadkar if (likely(ext4_mb_good_group(ac, grp->bb_group, 1))) 952196e402aSHarshad Shirwadkar break; 953196e402aSHarshad Shirwadkar } 954196e402aSHarshad Shirwadkar goto done; 955196e402aSHarshad Shirwadkar } 956196e402aSHarshad Shirwadkar 957196e402aSHarshad Shirwadkar node = sbi->s_mb_avg_fragment_size_root.rb_node; 958196e402aSHarshad Shirwadkar best_so_far = 0; 959196e402aSHarshad Shirwadkar found = NULL; 960196e402aSHarshad Shirwadkar 961196e402aSHarshad Shirwadkar while (node) { 962196e402aSHarshad Shirwadkar grp = rb_entry(node, struct ext4_group_info, 963196e402aSHarshad Shirwadkar bb_avg_fragment_size_rb); 964196e402aSHarshad Shirwadkar avg_fragment_size = 0; 965196e402aSHarshad Shirwadkar if (ext4_mb_good_group(ac, grp->bb_group, 1)) { 966196e402aSHarshad Shirwadkar avg_fragment_size = grp->bb_fragments ? 967196e402aSHarshad Shirwadkar grp->bb_free / grp->bb_fragments : 0; 968196e402aSHarshad Shirwadkar if (!best_so_far || avg_fragment_size < best_so_far) { 969196e402aSHarshad Shirwadkar best_so_far = avg_fragment_size; 970196e402aSHarshad Shirwadkar found = node; 971196e402aSHarshad Shirwadkar } 972196e402aSHarshad Shirwadkar } 973196e402aSHarshad Shirwadkar if (avg_fragment_size > ac->ac_g_ex.fe_len) 974196e402aSHarshad Shirwadkar node = node->rb_right; 975196e402aSHarshad Shirwadkar else 976196e402aSHarshad Shirwadkar node = node->rb_left; 977196e402aSHarshad Shirwadkar } 978196e402aSHarshad Shirwadkar 979196e402aSHarshad Shirwadkar done: 980196e402aSHarshad Shirwadkar if (found) { 981196e402aSHarshad Shirwadkar grp = rb_entry(found, struct ext4_group_info, 982196e402aSHarshad Shirwadkar bb_avg_fragment_size_rb); 983196e402aSHarshad Shirwadkar *group = grp->bb_group; 984196e402aSHarshad Shirwadkar ac->ac_flags |= EXT4_MB_CR1_OPTIMIZED; 985196e402aSHarshad Shirwadkar } else { 986196e402aSHarshad Shirwadkar *new_cr = 2; 987196e402aSHarshad Shirwadkar } 988196e402aSHarshad Shirwadkar 989196e402aSHarshad Shirwadkar read_unlock(&sbi->s_mb_rb_lock); 990196e402aSHarshad Shirwadkar ac->ac_last_optimal_group = *group; 991196e402aSHarshad Shirwadkar } 992196e402aSHarshad Shirwadkar 993196e402aSHarshad Shirwadkar static inline int should_optimize_scan(struct ext4_allocation_context *ac) 994196e402aSHarshad Shirwadkar { 995196e402aSHarshad Shirwadkar if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN))) 996196e402aSHarshad Shirwadkar return 0; 997196e402aSHarshad Shirwadkar if (ac->ac_criteria >= 2) 998196e402aSHarshad Shirwadkar return 0; 999196e402aSHarshad Shirwadkar if (ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) 1000196e402aSHarshad Shirwadkar return 0; 1001196e402aSHarshad Shirwadkar return 1; 1002196e402aSHarshad Shirwadkar } 1003196e402aSHarshad Shirwadkar 1004196e402aSHarshad Shirwadkar /* 1005196e402aSHarshad Shirwadkar * Return next linear group for allocation. If linear traversal should not be 1006196e402aSHarshad Shirwadkar * performed, this function just returns the same group 1007196e402aSHarshad Shirwadkar */ 1008196e402aSHarshad Shirwadkar static int 1009196e402aSHarshad Shirwadkar next_linear_group(struct ext4_allocation_context *ac, int group, int ngroups) 1010196e402aSHarshad Shirwadkar { 1011196e402aSHarshad Shirwadkar if (!should_optimize_scan(ac)) 1012196e402aSHarshad Shirwadkar goto inc_and_return; 1013196e402aSHarshad Shirwadkar 1014196e402aSHarshad Shirwadkar if (ac->ac_groups_linear_remaining) { 1015196e402aSHarshad Shirwadkar ac->ac_groups_linear_remaining--; 1016196e402aSHarshad Shirwadkar goto inc_and_return; 1017196e402aSHarshad Shirwadkar } 1018196e402aSHarshad Shirwadkar 1019196e402aSHarshad Shirwadkar if (ac->ac_flags & EXT4_MB_SEARCH_NEXT_LINEAR) { 1020196e402aSHarshad Shirwadkar ac->ac_flags &= ~EXT4_MB_SEARCH_NEXT_LINEAR; 1021196e402aSHarshad Shirwadkar goto inc_and_return; 1022196e402aSHarshad Shirwadkar } 1023196e402aSHarshad Shirwadkar 1024196e402aSHarshad Shirwadkar return group; 1025196e402aSHarshad Shirwadkar inc_and_return: 1026196e402aSHarshad Shirwadkar /* 1027196e402aSHarshad Shirwadkar * Artificially restricted ngroups for non-extent 1028196e402aSHarshad Shirwadkar * files makes group > ngroups possible on first loop. 1029196e402aSHarshad Shirwadkar */ 1030196e402aSHarshad Shirwadkar return group + 1 >= ngroups ? 0 : group + 1; 1031196e402aSHarshad Shirwadkar } 1032196e402aSHarshad Shirwadkar 1033196e402aSHarshad Shirwadkar /* 1034196e402aSHarshad Shirwadkar * ext4_mb_choose_next_group: choose next group for allocation. 1035196e402aSHarshad Shirwadkar * 1036196e402aSHarshad Shirwadkar * @ac Allocation Context 1037196e402aSHarshad Shirwadkar * @new_cr This is an output parameter. If the there is no good group 1038196e402aSHarshad Shirwadkar * available at current CR level, this field is updated to indicate 1039196e402aSHarshad Shirwadkar * the new cr level that should be used. 1040196e402aSHarshad Shirwadkar * @group This is an input / output parameter. As an input it indicates the 1041196e402aSHarshad Shirwadkar * next group that the allocator intends to use for allocation. As 1042196e402aSHarshad Shirwadkar * output, this field indicates the next group that should be used as 1043196e402aSHarshad Shirwadkar * determined by the optimization functions. 1044196e402aSHarshad Shirwadkar * @ngroups Total number of groups 1045196e402aSHarshad Shirwadkar */ 1046196e402aSHarshad Shirwadkar static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac, 1047196e402aSHarshad Shirwadkar int *new_cr, ext4_group_t *group, ext4_group_t ngroups) 1048196e402aSHarshad Shirwadkar { 1049196e402aSHarshad Shirwadkar *new_cr = ac->ac_criteria; 1050196e402aSHarshad Shirwadkar 1051196e402aSHarshad Shirwadkar if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) 1052196e402aSHarshad Shirwadkar return; 1053196e402aSHarshad Shirwadkar 1054196e402aSHarshad Shirwadkar if (*new_cr == 0) { 1055196e402aSHarshad Shirwadkar ext4_mb_choose_next_group_cr0(ac, new_cr, group, ngroups); 1056196e402aSHarshad Shirwadkar } else if (*new_cr == 1) { 1057196e402aSHarshad Shirwadkar ext4_mb_choose_next_group_cr1(ac, new_cr, group, ngroups); 1058196e402aSHarshad Shirwadkar } else { 1059196e402aSHarshad Shirwadkar /* 1060196e402aSHarshad Shirwadkar * TODO: For CR=2, we can arrange groups in an rb tree sorted by 1061196e402aSHarshad Shirwadkar * bb_free. But until that happens, we should never come here. 1062196e402aSHarshad Shirwadkar */ 1063196e402aSHarshad Shirwadkar WARN_ON(1); 1064196e402aSHarshad Shirwadkar } 1065196e402aSHarshad Shirwadkar } 1066196e402aSHarshad Shirwadkar 10678a57d9d6SCurt Wohlgemuth /* 10688a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 10698a57d9d6SCurt Wohlgemuth * group. 10708a57d9d6SCurt Wohlgemuth */ 10718a57d9d6SCurt Wohlgemuth static void 10728a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 10738a57d9d6SCurt Wohlgemuth { 1074196e402aSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 10758a57d9d6SCurt Wohlgemuth int i; 10768a57d9d6SCurt Wohlgemuth 1077196e402aSHarshad Shirwadkar if (test_opt2(sb, MB_OPTIMIZE_SCAN) && grp->bb_largest_free_order >= 0) { 1078196e402aSHarshad Shirwadkar write_lock(&sbi->s_mb_largest_free_orders_locks[ 1079196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1080196e402aSHarshad Shirwadkar list_del_init(&grp->bb_largest_free_order_node); 1081196e402aSHarshad Shirwadkar write_unlock(&sbi->s_mb_largest_free_orders_locks[ 1082196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1083196e402aSHarshad Shirwadkar } 10848a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = -1; /* uninit */ 10858a57d9d6SCurt Wohlgemuth 1086196e402aSHarshad Shirwadkar for (i = MB_NUM_ORDERS(sb) - 1; i >= 0; i--) { 10878a57d9d6SCurt Wohlgemuth if (grp->bb_counters[i] > 0) { 10888a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 10898a57d9d6SCurt Wohlgemuth break; 10908a57d9d6SCurt Wohlgemuth } 10918a57d9d6SCurt Wohlgemuth } 1092196e402aSHarshad Shirwadkar if (test_opt2(sb, MB_OPTIMIZE_SCAN) && 1093196e402aSHarshad Shirwadkar grp->bb_largest_free_order >= 0 && grp->bb_free) { 1094196e402aSHarshad Shirwadkar write_lock(&sbi->s_mb_largest_free_orders_locks[ 1095196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1096196e402aSHarshad Shirwadkar list_add_tail(&grp->bb_largest_free_order_node, 1097196e402aSHarshad Shirwadkar &sbi->s_mb_largest_free_orders[grp->bb_largest_free_order]); 1098196e402aSHarshad Shirwadkar write_unlock(&sbi->s_mb_largest_free_orders_locks[ 1099196e402aSHarshad Shirwadkar grp->bb_largest_free_order]); 1100196e402aSHarshad Shirwadkar } 11018a57d9d6SCurt Wohlgemuth } 11028a57d9d6SCurt Wohlgemuth 1103089ceeccSEric Sandeen static noinline_for_stack 1104089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 1105c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 1106c9de560dSAlex Tomas { 1107c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 1108e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 11097137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 1110a36b4498SEric Sandeen ext4_grpblk_t i = 0; 1111a36b4498SEric Sandeen ext4_grpblk_t first; 1112a36b4498SEric Sandeen ext4_grpblk_t len; 1113c9de560dSAlex Tomas unsigned free = 0; 1114c9de560dSAlex Tomas unsigned fragments = 0; 1115c9de560dSAlex Tomas unsigned long long period = get_cycles(); 1116c9de560dSAlex Tomas 1117c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 1118c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 1119ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 1120c9de560dSAlex Tomas grp->bb_first_free = i; 1121c9de560dSAlex Tomas while (i < max) { 1122c9de560dSAlex Tomas fragments++; 1123c9de560dSAlex Tomas first = i; 1124ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 1125c9de560dSAlex Tomas len = i - first; 1126c9de560dSAlex Tomas free += len; 1127c9de560dSAlex Tomas if (len > 1) 1128c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 1129c9de560dSAlex Tomas else 1130c9de560dSAlex Tomas grp->bb_counters[0]++; 1131c9de560dSAlex Tomas if (i < max) 1132ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 1133c9de560dSAlex Tomas } 1134c9de560dSAlex Tomas grp->bb_fragments = fragments; 1135c9de560dSAlex Tomas 1136c9de560dSAlex Tomas if (free != grp->bb_free) { 1137e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 113894d4c066STheodore Ts'o "block bitmap and bg descriptor " 113994d4c066STheodore Ts'o "inconsistent: %u vs %u free clusters", 1140e29136f8STheodore Ts'o free, grp->bb_free); 1141e56eb659SAneesh Kumar K.V /* 1142163a203dSDarrick J. Wong * If we intend to continue, we consider group descriptor 1143e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 1144e56eb659SAneesh Kumar K.V */ 1145c9de560dSAlex Tomas grp->bb_free = free; 1146db79e6d1SWang Shilong ext4_mark_group_bitmap_corrupted(sb, group, 1147db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 1148c9de560dSAlex Tomas } 11498a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 1150c9de560dSAlex Tomas 1151c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 1152c9de560dSAlex Tomas 1153c9de560dSAlex Tomas period = get_cycles() - period; 115467d25186SHarshad Shirwadkar atomic_inc(&sbi->s_mb_buddies_generated); 115567d25186SHarshad Shirwadkar atomic64_add(period, &sbi->s_mb_generation_time); 1156196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(sb, grp); 1157c9de560dSAlex Tomas } 1158c9de560dSAlex Tomas 1159c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 1160c9de560dSAlex Tomas * for convenience. The information regarding each group 1161c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 1162c9de560dSAlex Tomas * block bitmap and buddy information. The information are 1163c9de560dSAlex Tomas * stored in the inode as 1164c9de560dSAlex Tomas * 1165c9de560dSAlex Tomas * { page } 1166c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 1167c9de560dSAlex Tomas * 1168c9de560dSAlex Tomas * 1169c9de560dSAlex Tomas * one block each for bitmap and buddy information. 1170c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 1171ea1754a0SKirill A. Shutemov * contain blocks_per_page (PAGE_SIZE / blocksize) blocks. 1172c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 1173c9de560dSAlex Tomas * is blocks_per_page/2 11748a57d9d6SCurt Wohlgemuth * 11758a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 11768a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 1177c9de560dSAlex Tomas */ 1178c9de560dSAlex Tomas 1179adb7ef60SKonstantin Khlebnikov static int ext4_mb_init_cache(struct page *page, char *incore, gfp_t gfp) 1180c9de560dSAlex Tomas { 11818df9675fSTheodore Ts'o ext4_group_t ngroups; 1182c9de560dSAlex Tomas int blocksize; 1183c9de560dSAlex Tomas int blocks_per_page; 1184c9de560dSAlex Tomas int groups_per_page; 1185c9de560dSAlex Tomas int err = 0; 1186c9de560dSAlex Tomas int i; 1187813e5727STheodore Ts'o ext4_group_t first_group, group; 1188c9de560dSAlex Tomas int first_block; 1189c9de560dSAlex Tomas struct super_block *sb; 1190c9de560dSAlex Tomas struct buffer_head *bhs; 1191fa77dcfaSDarrick J. Wong struct buffer_head **bh = NULL; 1192c9de560dSAlex Tomas struct inode *inode; 1193c9de560dSAlex Tomas char *data; 1194c9de560dSAlex Tomas char *bitmap; 11959b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 1196c9de560dSAlex Tomas 1197c9de560dSAlex Tomas inode = page->mapping->host; 1198c9de560dSAlex Tomas sb = inode->i_sb; 11998df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 120093407472SFabian Frederick blocksize = i_blocksize(inode); 120109cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / blocksize; 1202c9de560dSAlex Tomas 1203d3df1453SRitesh Harjani mb_debug(sb, "init page %lu\n", page->index); 1204d3df1453SRitesh Harjani 1205c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 1206c9de560dSAlex Tomas if (groups_per_page == 0) 1207c9de560dSAlex Tomas groups_per_page = 1; 1208c9de560dSAlex Tomas 1209c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 1210c9de560dSAlex Tomas if (groups_per_page > 1) { 1211c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 1212adb7ef60SKonstantin Khlebnikov bh = kzalloc(i, gfp); 1213813e5727STheodore Ts'o if (bh == NULL) { 1214813e5727STheodore Ts'o err = -ENOMEM; 1215c9de560dSAlex Tomas goto out; 1216813e5727STheodore Ts'o } 1217c9de560dSAlex Tomas } else 1218c9de560dSAlex Tomas bh = &bhs; 1219c9de560dSAlex Tomas 1220c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 1221c9de560dSAlex Tomas 1222c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 1223813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 1224813e5727STheodore Ts'o if (group >= ngroups) 1225c9de560dSAlex Tomas break; 1226c9de560dSAlex Tomas 1227813e5727STheodore Ts'o grinfo = ext4_get_group_info(sb, group); 12289b8b7d35SAmir Goldstein /* 12299b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 12309b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 12319b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 12329b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 12339b8b7d35SAmir Goldstein */ 12349b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 12359b8b7d35SAmir Goldstein bh[i] = NULL; 12369b8b7d35SAmir Goldstein continue; 12379b8b7d35SAmir Goldstein } 1238cfd73237SAlex Zhuravlev bh[i] = ext4_read_block_bitmap_nowait(sb, group, false); 12399008a58eSDarrick J. Wong if (IS_ERR(bh[i])) { 12409008a58eSDarrick J. Wong err = PTR_ERR(bh[i]); 12419008a58eSDarrick J. Wong bh[i] = NULL; 1242c9de560dSAlex Tomas goto out; 12432ccb5fb9SAneesh Kumar K.V } 1244d3df1453SRitesh Harjani mb_debug(sb, "read bitmap for group %u\n", group); 1245c9de560dSAlex Tomas } 1246c9de560dSAlex Tomas 1247c9de560dSAlex Tomas /* wait for I/O completion */ 1248813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 12499008a58eSDarrick J. Wong int err2; 12509008a58eSDarrick J. Wong 12519008a58eSDarrick J. Wong if (!bh[i]) 12529008a58eSDarrick J. Wong continue; 12539008a58eSDarrick J. Wong err2 = ext4_wait_block_bitmap(sb, group, bh[i]); 12549008a58eSDarrick J. Wong if (!err) 12559008a58eSDarrick J. Wong err = err2; 1256813e5727STheodore Ts'o } 1257c9de560dSAlex Tomas 1258c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 1259c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 1260c9de560dSAlex Tomas group = (first_block + i) >> 1; 12618df9675fSTheodore Ts'o if (group >= ngroups) 1262c9de560dSAlex Tomas break; 1263c9de560dSAlex Tomas 12649b8b7d35SAmir Goldstein if (!bh[group - first_group]) 12659b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 12669b8b7d35SAmir Goldstein continue; 12679b8b7d35SAmir Goldstein 1268bbdc322fSLukas Czerner if (!buffer_verified(bh[group - first_group])) 1269bbdc322fSLukas Czerner /* Skip faulty bitmaps */ 1270bbdc322fSLukas Czerner continue; 1271bbdc322fSLukas Czerner err = 0; 1272bbdc322fSLukas Czerner 1273c9de560dSAlex Tomas /* 1274c9de560dSAlex Tomas * data carry information regarding this 1275c9de560dSAlex Tomas * particular group in the format specified 1276c9de560dSAlex Tomas * above 1277c9de560dSAlex Tomas * 1278c9de560dSAlex Tomas */ 1279c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 1280c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 1281c9de560dSAlex Tomas 1282c9de560dSAlex Tomas /* 1283c9de560dSAlex Tomas * We place the buddy block and bitmap block 1284c9de560dSAlex Tomas * close together 1285c9de560dSAlex Tomas */ 1286c9de560dSAlex Tomas if ((first_block + i) & 1) { 1287c9de560dSAlex Tomas /* this is block of buddy */ 1288c9de560dSAlex Tomas BUG_ON(incore == NULL); 1289d3df1453SRitesh Harjani mb_debug(sb, "put buddy for group %u in page %lu/%x\n", 1290c9de560dSAlex Tomas group, page->index, i * blocksize); 1291f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 1292c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 1293c9de560dSAlex Tomas grinfo->bb_fragments = 0; 1294c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 12951927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 12964b68f6dfSHarshad Shirwadkar (MB_NUM_ORDERS(sb))); 1297c9de560dSAlex Tomas /* 1298c9de560dSAlex Tomas * incore got set to the group block bitmap below 1299c9de560dSAlex Tomas */ 13007a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 13019b8b7d35SAmir Goldstein /* init the buddy */ 13029b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 1303c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 13047a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 1305c9de560dSAlex Tomas incore = NULL; 1306c9de560dSAlex Tomas } else { 1307c9de560dSAlex Tomas /* this is block of bitmap */ 1308c9de560dSAlex Tomas BUG_ON(incore != NULL); 1309d3df1453SRitesh Harjani mb_debug(sb, "put bitmap for group %u in page %lu/%x\n", 1310c9de560dSAlex Tomas group, page->index, i * blocksize); 1311f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 1312c9de560dSAlex Tomas 1313c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 1314c9de560dSAlex Tomas ext4_lock_group(sb, group); 1315c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 1316c9de560dSAlex Tomas 1317c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 1318c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 13197a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 1320c9de560dSAlex Tomas ext4_unlock_group(sb, group); 1321c9de560dSAlex Tomas 1322c9de560dSAlex Tomas /* set incore so that the buddy information can be 1323c9de560dSAlex Tomas * generated using this 1324c9de560dSAlex Tomas */ 1325c9de560dSAlex Tomas incore = data; 1326c9de560dSAlex Tomas } 1327c9de560dSAlex Tomas } 1328c9de560dSAlex Tomas SetPageUptodate(page); 1329c9de560dSAlex Tomas 1330c9de560dSAlex Tomas out: 1331c9de560dSAlex Tomas if (bh) { 13329b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 1333c9de560dSAlex Tomas brelse(bh[i]); 1334c9de560dSAlex Tomas if (bh != &bhs) 1335c9de560dSAlex Tomas kfree(bh); 1336c9de560dSAlex Tomas } 1337c9de560dSAlex Tomas return err; 1338c9de560dSAlex Tomas } 1339c9de560dSAlex Tomas 13408a57d9d6SCurt Wohlgemuth /* 13412de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 13422de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 13432de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 13442de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 1345eee4adc7SEric Sandeen */ 13462de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 1347adb7ef60SKonstantin Khlebnikov ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp) 1348eee4adc7SEric Sandeen { 13492de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 13502de8807bSAmir Goldstein int block, pnum, poff; 1351eee4adc7SEric Sandeen int blocks_per_page; 13522de8807bSAmir Goldstein struct page *page; 13532de8807bSAmir Goldstein 13542de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 13552de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 1356eee4adc7SEric Sandeen 135709cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1358eee4adc7SEric Sandeen /* 1359eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 1360eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 1361eee4adc7SEric Sandeen * So for each group we need two blocks. 1362eee4adc7SEric Sandeen */ 1363eee4adc7SEric Sandeen block = group * 2; 1364eee4adc7SEric Sandeen pnum = block / blocks_per_page; 13652de8807bSAmir Goldstein poff = block % blocks_per_page; 1366adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 13672de8807bSAmir Goldstein if (!page) 1368c57ab39bSYounger Liu return -ENOMEM; 13692de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 13702de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 13712de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1372eee4adc7SEric Sandeen 13732de8807bSAmir Goldstein if (blocks_per_page >= 2) { 13742de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 13752de8807bSAmir Goldstein return 0; 1376eee4adc7SEric Sandeen } 1377eee4adc7SEric Sandeen 13782de8807bSAmir Goldstein block++; 1379eee4adc7SEric Sandeen pnum = block / blocks_per_page; 1380adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 13812de8807bSAmir Goldstein if (!page) 1382c57ab39bSYounger Liu return -ENOMEM; 13832de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 13842de8807bSAmir Goldstein e4b->bd_buddy_page = page; 13852de8807bSAmir Goldstein return 0; 1386eee4adc7SEric Sandeen } 1387eee4adc7SEric Sandeen 13882de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 13892de8807bSAmir Goldstein { 13902de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 13912de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 139209cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 13932de8807bSAmir Goldstein } 13942de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 13952de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 139609cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 13972de8807bSAmir Goldstein } 1398eee4adc7SEric Sandeen } 1399eee4adc7SEric Sandeen 1400eee4adc7SEric Sandeen /* 14018a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 14028a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 14038a57d9d6SCurt Wohlgemuth * calling this routine! 14048a57d9d6SCurt Wohlgemuth */ 1405b6a758ecSAneesh Kumar K.V static noinline_for_stack 1406adb7ef60SKonstantin Khlebnikov int ext4_mb_init_group(struct super_block *sb, ext4_group_t group, gfp_t gfp) 1407b6a758ecSAneesh Kumar K.V { 1408b6a758ecSAneesh Kumar K.V 1409b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 14102de8807bSAmir Goldstein struct ext4_buddy e4b; 14112de8807bSAmir Goldstein struct page *page; 14122de8807bSAmir Goldstein int ret = 0; 1413b6a758ecSAneesh Kumar K.V 1414b10a44c3STheodore Ts'o might_sleep(); 1415d3df1453SRitesh Harjani mb_debug(sb, "init group %u\n", group); 1416b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 1417b6a758ecSAneesh Kumar K.V /* 141808c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 141908c3a813SAneesh Kumar K.V * page which map to the group from which we are already 142008c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 142108c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 14222de8807bSAmir Goldstein * would have pinned buddy page to page cache. 14232457aec6SMel Gorman * The call to ext4_mb_get_buddy_page_lock will mark the 14242457aec6SMel Gorman * page accessed. 1425b6a758ecSAneesh Kumar K.V */ 1426adb7ef60SKonstantin Khlebnikov ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp); 14272de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1428b6a758ecSAneesh Kumar K.V /* 1429b6a758ecSAneesh Kumar K.V * somebody initialized the group 1430b6a758ecSAneesh Kumar K.V * return without doing anything 1431b6a758ecSAneesh Kumar K.V */ 1432b6a758ecSAneesh Kumar K.V goto err; 1433b6a758ecSAneesh Kumar K.V } 14342de8807bSAmir Goldstein 14352de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1436adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 14372de8807bSAmir Goldstein if (ret) 1438b6a758ecSAneesh Kumar K.V goto err; 14392de8807bSAmir Goldstein if (!PageUptodate(page)) { 1440b6a758ecSAneesh Kumar K.V ret = -EIO; 1441b6a758ecSAneesh Kumar K.V goto err; 1442b6a758ecSAneesh Kumar K.V } 1443b6a758ecSAneesh Kumar K.V 14442de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1445b6a758ecSAneesh Kumar K.V /* 1446b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1447b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1448b6a758ecSAneesh Kumar K.V * init the buddy 1449b6a758ecSAneesh Kumar K.V */ 14502de8807bSAmir Goldstein ret = 0; 1451b6a758ecSAneesh Kumar K.V goto err; 1452b6a758ecSAneesh Kumar K.V } 14532de8807bSAmir Goldstein /* init buddy cache */ 14542de8807bSAmir Goldstein page = e4b.bd_buddy_page; 1455adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp); 14562de8807bSAmir Goldstein if (ret) 14572de8807bSAmir Goldstein goto err; 14582de8807bSAmir Goldstein if (!PageUptodate(page)) { 1459b6a758ecSAneesh Kumar K.V ret = -EIO; 1460b6a758ecSAneesh Kumar K.V goto err; 1461b6a758ecSAneesh Kumar K.V } 1462b6a758ecSAneesh Kumar K.V err: 14632de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1464b6a758ecSAneesh Kumar K.V return ret; 1465b6a758ecSAneesh Kumar K.V } 1466b6a758ecSAneesh Kumar K.V 14678a57d9d6SCurt Wohlgemuth /* 14688a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 14698a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 14708a57d9d6SCurt Wohlgemuth * calling this routine! 14718a57d9d6SCurt Wohlgemuth */ 14724ddfef7bSEric Sandeen static noinline_for_stack int 1473adb7ef60SKonstantin Khlebnikov ext4_mb_load_buddy_gfp(struct super_block *sb, ext4_group_t group, 1474adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b, gfp_t gfp) 1475c9de560dSAlex Tomas { 1476c9de560dSAlex Tomas int blocks_per_page; 1477c9de560dSAlex Tomas int block; 1478c9de560dSAlex Tomas int pnum; 1479c9de560dSAlex Tomas int poff; 1480c9de560dSAlex Tomas struct page *page; 1481fdf6c7a7SShen Feng int ret; 1482920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1483920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1484920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1485c9de560dSAlex Tomas 1486b10a44c3STheodore Ts'o might_sleep(); 1487d3df1453SRitesh Harjani mb_debug(sb, "load group %u\n", group); 1488c9de560dSAlex Tomas 148909cbfeafSKirill A. Shutemov blocks_per_page = PAGE_SIZE / sb->s_blocksize; 1490920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1491c9de560dSAlex Tomas 1492c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1493529da704STao Ma e4b->bd_info = grp; 1494c9de560dSAlex Tomas e4b->bd_sb = sb; 1495c9de560dSAlex Tomas e4b->bd_group = group; 1496c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1497c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1498c9de560dSAlex Tomas 1499f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1500f41c0750SAneesh Kumar K.V /* 1501f41c0750SAneesh Kumar K.V * we need full data about the group 1502f41c0750SAneesh Kumar K.V * to make a good selection 1503f41c0750SAneesh Kumar K.V */ 1504adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, gfp); 1505f41c0750SAneesh Kumar K.V if (ret) 1506f41c0750SAneesh Kumar K.V return ret; 1507f41c0750SAneesh Kumar K.V } 1508f41c0750SAneesh Kumar K.V 1509c9de560dSAlex Tomas /* 1510c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1511c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1512c9de560dSAlex Tomas * So for each group we need two blocks. 1513c9de560dSAlex Tomas */ 1514c9de560dSAlex Tomas block = group * 2; 1515c9de560dSAlex Tomas pnum = block / blocks_per_page; 1516c9de560dSAlex Tomas poff = block % blocks_per_page; 1517c9de560dSAlex Tomas 1518c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1519c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 15202457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1521c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1522c9de560dSAlex Tomas if (page) 1523920313a7SAneesh Kumar K.V /* 1524920313a7SAneesh Kumar K.V * drop the page reference and try 1525920313a7SAneesh Kumar K.V * to get the page with lock. If we 1526920313a7SAneesh Kumar K.V * are not uptodate that implies 1527920313a7SAneesh Kumar K.V * somebody just created the page but 1528920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1529920313a7SAneesh Kumar K.V * wait for it to initialize. 1530920313a7SAneesh Kumar K.V */ 153109cbfeafSKirill A. Shutemov put_page(page); 1532adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1533c9de560dSAlex Tomas if (page) { 1534c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1535c9de560dSAlex Tomas if (!PageUptodate(page)) { 1536adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, NULL, gfp); 1537fdf6c7a7SShen Feng if (ret) { 1538fdf6c7a7SShen Feng unlock_page(page); 1539fdf6c7a7SShen Feng goto err; 1540fdf6c7a7SShen Feng } 1541c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1542c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1543c9de560dSAlex Tomas } 1544c9de560dSAlex Tomas unlock_page(page); 1545c9de560dSAlex Tomas } 1546c9de560dSAlex Tomas } 1547c57ab39bSYounger Liu if (page == NULL) { 1548c57ab39bSYounger Liu ret = -ENOMEM; 1549c57ab39bSYounger Liu goto err; 1550c57ab39bSYounger Liu } 1551c57ab39bSYounger Liu if (!PageUptodate(page)) { 1552fdf6c7a7SShen Feng ret = -EIO; 1553c9de560dSAlex Tomas goto err; 1554fdf6c7a7SShen Feng } 15552457aec6SMel Gorman 15562457aec6SMel Gorman /* Pages marked accessed already */ 1557c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1558c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1559c9de560dSAlex Tomas 1560c9de560dSAlex Tomas block++; 1561c9de560dSAlex Tomas pnum = block / blocks_per_page; 1562c9de560dSAlex Tomas poff = block % blocks_per_page; 1563c9de560dSAlex Tomas 15642457aec6SMel Gorman page = find_get_page_flags(inode->i_mapping, pnum, FGP_ACCESSED); 1565c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1566c9de560dSAlex Tomas if (page) 156709cbfeafSKirill A. Shutemov put_page(page); 1568adb7ef60SKonstantin Khlebnikov page = find_or_create_page(inode->i_mapping, pnum, gfp); 1569c9de560dSAlex Tomas if (page) { 1570c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1571fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1572adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_cache(page, e4b->bd_bitmap, 1573adb7ef60SKonstantin Khlebnikov gfp); 1574fdf6c7a7SShen Feng if (ret) { 1575fdf6c7a7SShen Feng unlock_page(page); 1576fdf6c7a7SShen Feng goto err; 1577fdf6c7a7SShen Feng } 1578fdf6c7a7SShen Feng } 1579c9de560dSAlex Tomas unlock_page(page); 1580c9de560dSAlex Tomas } 1581c9de560dSAlex Tomas } 1582c57ab39bSYounger Liu if (page == NULL) { 1583c57ab39bSYounger Liu ret = -ENOMEM; 1584c57ab39bSYounger Liu goto err; 1585c57ab39bSYounger Liu } 1586c57ab39bSYounger Liu if (!PageUptodate(page)) { 1587fdf6c7a7SShen Feng ret = -EIO; 1588c9de560dSAlex Tomas goto err; 1589fdf6c7a7SShen Feng } 15902457aec6SMel Gorman 15912457aec6SMel Gorman /* Pages marked accessed already */ 1592c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1593c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1594c9de560dSAlex Tomas 1595c9de560dSAlex Tomas return 0; 1596c9de560dSAlex Tomas 1597c9de560dSAlex Tomas err: 159826626f11SYang Ruirui if (page) 159909cbfeafSKirill A. Shutemov put_page(page); 1600c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 160109cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1602c9de560dSAlex Tomas if (e4b->bd_buddy_page) 160309cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1604c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1605c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1606fdf6c7a7SShen Feng return ret; 1607c9de560dSAlex Tomas } 1608c9de560dSAlex Tomas 1609adb7ef60SKonstantin Khlebnikov static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1610adb7ef60SKonstantin Khlebnikov struct ext4_buddy *e4b) 1611adb7ef60SKonstantin Khlebnikov { 1612adb7ef60SKonstantin Khlebnikov return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS); 1613adb7ef60SKonstantin Khlebnikov } 1614adb7ef60SKonstantin Khlebnikov 1615e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1616c9de560dSAlex Tomas { 1617c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 161809cbfeafSKirill A. Shutemov put_page(e4b->bd_bitmap_page); 1619c9de560dSAlex Tomas if (e4b->bd_buddy_page) 162009cbfeafSKirill A. Shutemov put_page(e4b->bd_buddy_page); 1621c9de560dSAlex Tomas } 1622c9de560dSAlex Tomas 1623c9de560dSAlex Tomas 1624c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1625c9de560dSAlex Tomas { 1626ce3cca33SChunguang Xu int order = 1, max; 1627c9de560dSAlex Tomas void *bb; 1628c9de560dSAlex Tomas 1629c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 1630c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1631c9de560dSAlex Tomas 1632c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1633ce3cca33SChunguang Xu bb = mb_find_buddy(e4b, order, &max); 1634ce3cca33SChunguang Xu if (!mb_test_bit(block >> order, bb)) { 1635c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1636c9de560dSAlex Tomas return order; 1637c9de560dSAlex Tomas } 1638c9de560dSAlex Tomas order++; 1639c9de560dSAlex Tomas } 1640c9de560dSAlex Tomas return 0; 1641c9de560dSAlex Tomas } 1642c9de560dSAlex Tomas 1643955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1644c9de560dSAlex Tomas { 1645c9de560dSAlex Tomas __u32 *addr; 1646c9de560dSAlex Tomas 1647c9de560dSAlex Tomas len = cur + len; 1648c9de560dSAlex Tomas while (cur < len) { 1649c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1650c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1651c9de560dSAlex Tomas addr = bm + (cur >> 3); 1652c9de560dSAlex Tomas *addr = 0; 1653c9de560dSAlex Tomas cur += 32; 1654c9de560dSAlex Tomas continue; 1655c9de560dSAlex Tomas } 1656e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1657c9de560dSAlex Tomas cur++; 1658c9de560dSAlex Tomas } 1659c9de560dSAlex Tomas } 1660c9de560dSAlex Tomas 1661eabe0444SAndrey Sidorov /* clear bits in given range 1662eabe0444SAndrey Sidorov * will return first found zero bit if any, -1 otherwise 1663eabe0444SAndrey Sidorov */ 1664eabe0444SAndrey Sidorov static int mb_test_and_clear_bits(void *bm, int cur, int len) 1665eabe0444SAndrey Sidorov { 1666eabe0444SAndrey Sidorov __u32 *addr; 1667eabe0444SAndrey Sidorov int zero_bit = -1; 1668eabe0444SAndrey Sidorov 1669eabe0444SAndrey Sidorov len = cur + len; 1670eabe0444SAndrey Sidorov while (cur < len) { 1671eabe0444SAndrey Sidorov if ((cur & 31) == 0 && (len - cur) >= 32) { 1672eabe0444SAndrey Sidorov /* fast path: clear whole word at once */ 1673eabe0444SAndrey Sidorov addr = bm + (cur >> 3); 1674eabe0444SAndrey Sidorov if (*addr != (__u32)(-1) && zero_bit == -1) 1675eabe0444SAndrey Sidorov zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); 1676eabe0444SAndrey Sidorov *addr = 0; 1677eabe0444SAndrey Sidorov cur += 32; 1678eabe0444SAndrey Sidorov continue; 1679eabe0444SAndrey Sidorov } 1680eabe0444SAndrey Sidorov if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) 1681eabe0444SAndrey Sidorov zero_bit = cur; 1682eabe0444SAndrey Sidorov cur++; 1683eabe0444SAndrey Sidorov } 1684eabe0444SAndrey Sidorov 1685eabe0444SAndrey Sidorov return zero_bit; 1686eabe0444SAndrey Sidorov } 1687eabe0444SAndrey Sidorov 1688c3e94d1dSYongqiang Yang void ext4_set_bits(void *bm, int cur, int len) 1689c9de560dSAlex Tomas { 1690c9de560dSAlex Tomas __u32 *addr; 1691c9de560dSAlex Tomas 1692c9de560dSAlex Tomas len = cur + len; 1693c9de560dSAlex Tomas while (cur < len) { 1694c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1695c9de560dSAlex Tomas /* fast path: set whole word at once */ 1696c9de560dSAlex Tomas addr = bm + (cur >> 3); 1697c9de560dSAlex Tomas *addr = 0xffffffff; 1698c9de560dSAlex Tomas cur += 32; 1699c9de560dSAlex Tomas continue; 1700c9de560dSAlex Tomas } 1701e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1702c9de560dSAlex Tomas cur++; 1703c9de560dSAlex Tomas } 1704c9de560dSAlex Tomas } 1705c9de560dSAlex Tomas 1706eabe0444SAndrey Sidorov static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) 1707eabe0444SAndrey Sidorov { 1708eabe0444SAndrey Sidorov if (mb_test_bit(*bit + side, bitmap)) { 1709eabe0444SAndrey Sidorov mb_clear_bit(*bit, bitmap); 1710eabe0444SAndrey Sidorov (*bit) -= side; 1711eabe0444SAndrey Sidorov return 1; 1712eabe0444SAndrey Sidorov } 1713eabe0444SAndrey Sidorov else { 1714eabe0444SAndrey Sidorov (*bit) += side; 1715eabe0444SAndrey Sidorov mb_set_bit(*bit, bitmap); 1716eabe0444SAndrey Sidorov return -1; 1717eabe0444SAndrey Sidorov } 1718eabe0444SAndrey Sidorov } 1719eabe0444SAndrey Sidorov 1720eabe0444SAndrey Sidorov static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) 1721eabe0444SAndrey Sidorov { 1722eabe0444SAndrey Sidorov int max; 1723eabe0444SAndrey Sidorov int order = 1; 1724eabe0444SAndrey Sidorov void *buddy = mb_find_buddy(e4b, order, &max); 1725eabe0444SAndrey Sidorov 1726eabe0444SAndrey Sidorov while (buddy) { 1727eabe0444SAndrey Sidorov void *buddy2; 1728eabe0444SAndrey Sidorov 1729eabe0444SAndrey Sidorov /* Bits in range [first; last] are known to be set since 1730eabe0444SAndrey Sidorov * corresponding blocks were allocated. Bits in range 1731eabe0444SAndrey Sidorov * (first; last) will stay set because they form buddies on 1732eabe0444SAndrey Sidorov * upper layer. We just deal with borders if they don't 1733eabe0444SAndrey Sidorov * align with upper layer and then go up. 1734eabe0444SAndrey Sidorov * Releasing entire group is all about clearing 1735eabe0444SAndrey Sidorov * single bit of highest order buddy. 1736eabe0444SAndrey Sidorov */ 1737eabe0444SAndrey Sidorov 1738eabe0444SAndrey Sidorov /* Example: 1739eabe0444SAndrey Sidorov * --------------------------------- 1740eabe0444SAndrey Sidorov * | 1 | 1 | 1 | 1 | 1741eabe0444SAndrey Sidorov * --------------------------------- 1742eabe0444SAndrey Sidorov * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1743eabe0444SAndrey Sidorov * --------------------------------- 1744eabe0444SAndrey Sidorov * 0 1 2 3 4 5 6 7 1745eabe0444SAndrey Sidorov * \_____________________/ 1746eabe0444SAndrey Sidorov * 1747eabe0444SAndrey Sidorov * Neither [1] nor [6] is aligned to above layer. 1748eabe0444SAndrey Sidorov * Left neighbour [0] is free, so mark it busy, 1749eabe0444SAndrey Sidorov * decrease bb_counters and extend range to 1750eabe0444SAndrey Sidorov * [0; 6] 1751eabe0444SAndrey Sidorov * Right neighbour [7] is busy. It can't be coaleasced with [6], so 1752eabe0444SAndrey Sidorov * mark [6] free, increase bb_counters and shrink range to 1753eabe0444SAndrey Sidorov * [0; 5]. 1754eabe0444SAndrey Sidorov * Then shift range to [0; 2], go up and do the same. 1755eabe0444SAndrey Sidorov */ 1756eabe0444SAndrey Sidorov 1757eabe0444SAndrey Sidorov 1758eabe0444SAndrey Sidorov if (first & 1) 1759eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); 1760eabe0444SAndrey Sidorov if (!(last & 1)) 1761eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); 1762eabe0444SAndrey Sidorov if (first > last) 1763eabe0444SAndrey Sidorov break; 1764eabe0444SAndrey Sidorov order++; 1765eabe0444SAndrey Sidorov 1766eabe0444SAndrey Sidorov if (first == last || !(buddy2 = mb_find_buddy(e4b, order, &max))) { 1767eabe0444SAndrey Sidorov mb_clear_bits(buddy, first, last - first + 1); 1768eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order - 1] += last - first + 1; 1769eabe0444SAndrey Sidorov break; 1770eabe0444SAndrey Sidorov } 1771eabe0444SAndrey Sidorov first >>= 1; 1772eabe0444SAndrey Sidorov last >>= 1; 1773eabe0444SAndrey Sidorov buddy = buddy2; 1774eabe0444SAndrey Sidorov } 1775eabe0444SAndrey Sidorov } 1776eabe0444SAndrey Sidorov 17777e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1778c9de560dSAlex Tomas int first, int count) 1779c9de560dSAlex Tomas { 1780eabe0444SAndrey Sidorov int left_is_free = 0; 1781eabe0444SAndrey Sidorov int right_is_free = 0; 1782eabe0444SAndrey Sidorov int block; 1783eabe0444SAndrey Sidorov int last = first + count - 1; 1784c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1785c9de560dSAlex Tomas 1786c99d1e6eSTheodore Ts'o if (WARN_ON(count == 0)) 1787c99d1e6eSTheodore Ts'o return; 1788eabe0444SAndrey Sidorov BUG_ON(last >= (sb->s_blocksize << 3)); 1789bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1790163a203dSDarrick J. Wong /* Don't bother if the block group is corrupt. */ 1791163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) 1792163a203dSDarrick J. Wong return; 1793163a203dSDarrick J. Wong 1794c9de560dSAlex Tomas mb_check_buddy(e4b); 1795c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1796c9de560dSAlex Tomas 179707b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 1798c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1799c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1800c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1801c9de560dSAlex Tomas 1802eabe0444SAndrey Sidorov /* access memory sequentially: check left neighbour, 1803eabe0444SAndrey Sidorov * clear range and then check right neighbour 1804eabe0444SAndrey Sidorov */ 1805c9de560dSAlex Tomas if (first != 0) 1806eabe0444SAndrey Sidorov left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); 1807eabe0444SAndrey Sidorov block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); 1808eabe0444SAndrey Sidorov if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) 1809eabe0444SAndrey Sidorov right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); 1810c9de560dSAlex Tomas 1811eabe0444SAndrey Sidorov if (unlikely(block != -1)) { 1812e43bb4e6SNamjae Jeon struct ext4_sb_info *sbi = EXT4_SB(sb); 1813c9de560dSAlex Tomas ext4_fsblk_t blocknr; 18145661bd68SAkinobu Mita 18155661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 181649598e04SJun Piao blocknr += EXT4_C2B(sbi, block); 18178016e29fSHarshad Shirwadkar if (!(sbi->s_mount_state & EXT4_FC_REPLAY)) { 18185d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1819e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1820e29136f8STheodore Ts'o blocknr, 18218016e29fSHarshad Shirwadkar "freeing already freed block (bit %u); block bitmap corrupt.", 1822163a203dSDarrick J. Wong block); 18238016e29fSHarshad Shirwadkar ext4_mark_group_bitmap_corrupted( 18248016e29fSHarshad Shirwadkar sb, e4b->bd_group, 1825db79e6d1SWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 18268016e29fSHarshad Shirwadkar } 1827eabe0444SAndrey Sidorov goto done; 1828c9de560dSAlex Tomas } 1829c9de560dSAlex Tomas 1830eabe0444SAndrey Sidorov /* let's maintain fragments counter */ 1831eabe0444SAndrey Sidorov if (left_is_free && right_is_free) 1832eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments--; 1833eabe0444SAndrey Sidorov else if (!left_is_free && !right_is_free) 1834eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments++; 1835c9de560dSAlex Tomas 1836eabe0444SAndrey Sidorov /* buddy[0] == bd_bitmap is a special case, so handle 1837eabe0444SAndrey Sidorov * it right away and let mb_buddy_mark_free stay free of 1838eabe0444SAndrey Sidorov * zero order checks. 1839eabe0444SAndrey Sidorov * Check if neighbours are to be coaleasced, 1840eabe0444SAndrey Sidorov * adjust bitmap bb_counters and borders appropriately. 1841eabe0444SAndrey Sidorov */ 1842eabe0444SAndrey Sidorov if (first & 1) { 1843eabe0444SAndrey Sidorov first += !left_is_free; 1844eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; 1845c9de560dSAlex Tomas } 1846eabe0444SAndrey Sidorov if (!(last & 1)) { 1847eabe0444SAndrey Sidorov last -= !right_is_free; 1848eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; 1849c9de560dSAlex Tomas } 1850eabe0444SAndrey Sidorov 1851eabe0444SAndrey Sidorov if (first <= last) 1852eabe0444SAndrey Sidorov mb_buddy_mark_free(e4b, first >> 1, last >> 1); 1853eabe0444SAndrey Sidorov 1854eabe0444SAndrey Sidorov done: 18558a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1856196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(sb, e4b->bd_info); 1857c9de560dSAlex Tomas mb_check_buddy(e4b); 1858c9de560dSAlex Tomas } 1859c9de560dSAlex Tomas 186015c006a2SRobin Dong static int mb_find_extent(struct ext4_buddy *e4b, int block, 1861c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1862c9de560dSAlex Tomas { 1863c9de560dSAlex Tomas int next = block; 186415c006a2SRobin Dong int max, order; 1865c9de560dSAlex Tomas void *buddy; 1866c9de560dSAlex Tomas 1867bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1868c9de560dSAlex Tomas BUG_ON(ex == NULL); 1869c9de560dSAlex Tomas 187015c006a2SRobin Dong buddy = mb_find_buddy(e4b, 0, &max); 1871c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1872c9de560dSAlex Tomas BUG_ON(block >= max); 1873c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1874c9de560dSAlex Tomas ex->fe_len = 0; 1875c9de560dSAlex Tomas ex->fe_start = 0; 1876c9de560dSAlex Tomas ex->fe_group = 0; 1877c9de560dSAlex Tomas return 0; 1878c9de560dSAlex Tomas } 1879c9de560dSAlex Tomas 1880c9de560dSAlex Tomas /* find actual order */ 1881c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1882c9de560dSAlex Tomas block = block >> order; 1883c9de560dSAlex Tomas 1884c9de560dSAlex Tomas ex->fe_len = 1 << order; 1885c9de560dSAlex Tomas ex->fe_start = block << order; 1886c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1887c9de560dSAlex Tomas 1888c9de560dSAlex Tomas /* calc difference from given start */ 1889c9de560dSAlex Tomas next = next - ex->fe_start; 1890c9de560dSAlex Tomas ex->fe_len -= next; 1891c9de560dSAlex Tomas ex->fe_start += next; 1892c9de560dSAlex Tomas 1893c9de560dSAlex Tomas while (needed > ex->fe_len && 1894d8ec0c39SAlan Cox mb_find_buddy(e4b, order, &max)) { 1895c9de560dSAlex Tomas 1896c9de560dSAlex Tomas if (block + 1 >= max) 1897c9de560dSAlex Tomas break; 1898c9de560dSAlex Tomas 1899c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1900c5e8f3f3STheodore Ts'o if (mb_test_bit(next, e4b->bd_bitmap)) 1901c9de560dSAlex Tomas break; 1902c9de560dSAlex Tomas 1903b051d8dcSRobin Dong order = mb_find_order_for_block(e4b, next); 1904c9de560dSAlex Tomas 1905c9de560dSAlex Tomas block = next >> order; 1906c9de560dSAlex Tomas ex->fe_len += 1 << order; 1907c9de560dSAlex Tomas } 1908c9de560dSAlex Tomas 190931562b95SJan Kara if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) { 191043c73221STheodore Ts'o /* Should never happen! (but apparently sometimes does?!?) */ 191143c73221STheodore Ts'o WARN_ON(1); 191243c73221STheodore Ts'o ext4_error(e4b->bd_sb, "corruption or bug in mb_find_extent " 191343c73221STheodore Ts'o "block=%d, order=%d needed=%d ex=%u/%d/%d@%u", 191443c73221STheodore Ts'o block, order, needed, ex->fe_group, ex->fe_start, 191543c73221STheodore Ts'o ex->fe_len, ex->fe_logical); 191643c73221STheodore Ts'o ex->fe_len = 0; 191743c73221STheodore Ts'o ex->fe_start = 0; 191843c73221STheodore Ts'o ex->fe_group = 0; 191943c73221STheodore Ts'o } 1920c9de560dSAlex Tomas return ex->fe_len; 1921c9de560dSAlex Tomas } 1922c9de560dSAlex Tomas 1923c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1924c9de560dSAlex Tomas { 1925c9de560dSAlex Tomas int ord; 1926c9de560dSAlex Tomas int mlen = 0; 1927c9de560dSAlex Tomas int max = 0; 1928c9de560dSAlex Tomas int cur; 1929c9de560dSAlex Tomas int start = ex->fe_start; 1930c9de560dSAlex Tomas int len = ex->fe_len; 1931c9de560dSAlex Tomas unsigned ret = 0; 1932c9de560dSAlex Tomas int len0 = len; 1933c9de560dSAlex Tomas void *buddy; 1934c9de560dSAlex Tomas 1935c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1936c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1937bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1938c9de560dSAlex Tomas mb_check_buddy(e4b); 1939c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1940c9de560dSAlex Tomas 194107b5b8e1SRitesh Harjani this_cpu_inc(discard_pa_seq); 1942c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1943c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1944c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1945c9de560dSAlex Tomas 1946c9de560dSAlex Tomas /* let's maintain fragments counter */ 1947c9de560dSAlex Tomas if (start != 0) 1948c5e8f3f3STheodore Ts'o mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); 1949c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1950c5e8f3f3STheodore Ts'o max = !mb_test_bit(start + len, e4b->bd_bitmap); 1951c9de560dSAlex Tomas if (mlen && max) 1952c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1953c9de560dSAlex Tomas else if (!mlen && !max) 1954c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1955c9de560dSAlex Tomas 1956c9de560dSAlex Tomas /* let's maintain buddy itself */ 1957c9de560dSAlex Tomas while (len) { 1958c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1959c9de560dSAlex Tomas 1960c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1961c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1962c9de560dSAlex Tomas mlen = 1 << ord; 1963c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1964c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1965c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1966c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1967c9de560dSAlex Tomas start += mlen; 1968c9de560dSAlex Tomas len -= mlen; 1969c9de560dSAlex Tomas BUG_ON(len < 0); 1970c9de560dSAlex Tomas continue; 1971c9de560dSAlex Tomas } 1972c9de560dSAlex Tomas 1973c9de560dSAlex Tomas /* store for history */ 1974c9de560dSAlex Tomas if (ret == 0) 1975c9de560dSAlex Tomas ret = len | (ord << 16); 1976c9de560dSAlex Tomas 1977c9de560dSAlex Tomas /* we have to split large buddy */ 1978c9de560dSAlex Tomas BUG_ON(ord <= 0); 1979c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1980c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1981c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1982c9de560dSAlex Tomas 1983c9de560dSAlex Tomas ord--; 1984c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1985c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1986c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1987c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1988c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1989c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1990c9de560dSAlex Tomas } 19918a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 1992c9de560dSAlex Tomas 1993196e402aSHarshad Shirwadkar mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info); 1994c5e8f3f3STheodore Ts'o ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0); 1995c9de560dSAlex Tomas mb_check_buddy(e4b); 1996c9de560dSAlex Tomas 1997c9de560dSAlex Tomas return ret; 1998c9de560dSAlex Tomas } 1999c9de560dSAlex Tomas 2000c9de560dSAlex Tomas /* 2001c9de560dSAlex Tomas * Must be called under group lock! 2002c9de560dSAlex Tomas */ 2003c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 2004c9de560dSAlex Tomas struct ext4_buddy *e4b) 2005c9de560dSAlex Tomas { 2006c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2007c9de560dSAlex Tomas int ret; 2008c9de560dSAlex Tomas 2009c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 2010c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2011c9de560dSAlex Tomas 2012c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 2013c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 2014c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 2015c9de560dSAlex Tomas 2016c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 2017c9de560dSAlex Tomas * allocated blocks for history */ 2018c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 2019c9de560dSAlex Tomas 2020c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 2021c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 2022c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 2023c9de560dSAlex Tomas 2024c3a326a6SAneesh Kumar K.V /* 2025c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 2026c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 2027c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 2028c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 2029c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 2030c3a326a6SAneesh Kumar K.V */ 2031c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 2032c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 2033c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 2034c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 2035c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 20364ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2037c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2038c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 2039c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 2040c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2041c9de560dSAlex Tomas } 204253f86b17SRitesh Harjani /* 204353f86b17SRitesh Harjani * As we've just preallocated more space than 204453f86b17SRitesh Harjani * user requested originally, we store allocated 204553f86b17SRitesh Harjani * space in a special descriptor. 204653f86b17SRitesh Harjani */ 204753f86b17SRitesh Harjani if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 204853f86b17SRitesh Harjani ext4_mb_new_preallocation(ac); 204953f86b17SRitesh Harjani 2050c9de560dSAlex Tomas } 2051c9de560dSAlex Tomas 2052c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 2053c9de560dSAlex Tomas struct ext4_buddy *e4b, 2054c9de560dSAlex Tomas int finish_group) 2055c9de560dSAlex Tomas { 2056c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2057c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 2058c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 2059c9de560dSAlex Tomas struct ext4_free_extent ex; 2060c9de560dSAlex Tomas int max; 2061c9de560dSAlex Tomas 2062032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 2063032115fcSAneesh Kumar K.V return; 2064c9de560dSAlex Tomas /* 2065c9de560dSAlex Tomas * We don't want to scan for a whole year 2066c9de560dSAlex Tomas */ 2067c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 2068c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2069c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 2070c9de560dSAlex Tomas return; 2071c9de560dSAlex Tomas } 2072c9de560dSAlex Tomas 2073c9de560dSAlex Tomas /* 2074c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 2075c9de560dSAlex Tomas */ 2076c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 2077c9de560dSAlex Tomas return; 2078c9de560dSAlex Tomas 2079c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 2080c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 2081c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 2082c9de560dSAlex Tomas * when it was found (within this lock-unlock 2083c9de560dSAlex Tomas * period or not) */ 208415c006a2SRobin Dong max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex); 2085c9de560dSAlex Tomas if (max >= gex->fe_len) { 2086c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2087c9de560dSAlex Tomas return; 2088c9de560dSAlex Tomas } 2089c9de560dSAlex Tomas } 2090c9de560dSAlex Tomas } 2091c9de560dSAlex Tomas 2092c9de560dSAlex Tomas /* 2093c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 2094c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 2095c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 2096c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 2097c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 2098c9de560dSAlex Tomas * mballoc can't find good enough extent. 2099c9de560dSAlex Tomas * 2100c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 2101c9de560dSAlex Tomas */ 2102c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 2103c9de560dSAlex Tomas struct ext4_free_extent *ex, 2104c9de560dSAlex Tomas struct ext4_buddy *e4b) 2105c9de560dSAlex Tomas { 2106c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 2107c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 2108c9de560dSAlex Tomas 2109c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 21107137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 21117137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 2112c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 2113c9de560dSAlex Tomas 2114c9de560dSAlex Tomas ac->ac_found++; 2115c9de560dSAlex Tomas 2116c9de560dSAlex Tomas /* 2117c9de560dSAlex Tomas * The special case - take what you catch first 2118c9de560dSAlex Tomas */ 2119c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2120c9de560dSAlex Tomas *bex = *ex; 2121c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2122c9de560dSAlex Tomas return; 2123c9de560dSAlex Tomas } 2124c9de560dSAlex Tomas 2125c9de560dSAlex Tomas /* 2126c9de560dSAlex Tomas * Let's check whether the chuck is good enough 2127c9de560dSAlex Tomas */ 2128c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 2129c9de560dSAlex Tomas *bex = *ex; 2130c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2131c9de560dSAlex Tomas return; 2132c9de560dSAlex Tomas } 2133c9de560dSAlex Tomas 2134c9de560dSAlex Tomas /* 2135c9de560dSAlex Tomas * If this is first found extent, just store it in the context 2136c9de560dSAlex Tomas */ 2137c9de560dSAlex Tomas if (bex->fe_len == 0) { 2138c9de560dSAlex Tomas *bex = *ex; 2139c9de560dSAlex Tomas return; 2140c9de560dSAlex Tomas } 2141c9de560dSAlex Tomas 2142c9de560dSAlex Tomas /* 2143c9de560dSAlex Tomas * If new found extent is better, store it in the context 2144c9de560dSAlex Tomas */ 2145c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 2146c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 2147c9de560dSAlex Tomas * larger than previous best one is better */ 2148c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 2149c9de560dSAlex Tomas *bex = *ex; 2150c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 2151c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 2152c9de560dSAlex Tomas * an extent that still satisfy the request, but is 2153c9de560dSAlex Tomas * smaller than previous one */ 2154c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 2155c9de560dSAlex Tomas *bex = *ex; 2156c9de560dSAlex Tomas } 2157c9de560dSAlex Tomas 2158c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 2159c9de560dSAlex Tomas } 2160c9de560dSAlex Tomas 2161089ceeccSEric Sandeen static noinline_for_stack 2162089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 2163c9de560dSAlex Tomas struct ext4_buddy *e4b) 2164c9de560dSAlex Tomas { 2165c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 2166c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 2167c9de560dSAlex Tomas int max; 2168c9de560dSAlex Tomas int err; 2169c9de560dSAlex Tomas 2170c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2171c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 2172c9de560dSAlex Tomas if (err) 2173c9de560dSAlex Tomas return err; 2174c9de560dSAlex Tomas 2175c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 217615c006a2SRobin Dong max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); 2177c9de560dSAlex Tomas 2178c9de560dSAlex Tomas if (max > 0) { 2179c9de560dSAlex Tomas ac->ac_b_ex = ex; 2180c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2181c9de560dSAlex Tomas } 2182c9de560dSAlex Tomas 2183c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 2184e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 2185c9de560dSAlex Tomas 2186c9de560dSAlex Tomas return 0; 2187c9de560dSAlex Tomas } 2188c9de560dSAlex Tomas 2189089ceeccSEric Sandeen static noinline_for_stack 2190089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 2191c9de560dSAlex Tomas struct ext4_buddy *e4b) 2192c9de560dSAlex Tomas { 2193c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 2194c9de560dSAlex Tomas int max; 2195c9de560dSAlex Tomas int err; 2196c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2197838cd0cfSYongqiang Yang struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2198c9de560dSAlex Tomas struct ext4_free_extent ex; 2199c9de560dSAlex Tomas 2200c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 2201c9de560dSAlex Tomas return 0; 2202838cd0cfSYongqiang Yang if (grp->bb_free == 0) 2203838cd0cfSYongqiang Yang return 0; 2204c9de560dSAlex Tomas 2205c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 2206c9de560dSAlex Tomas if (err) 2207c9de560dSAlex Tomas return err; 2208c9de560dSAlex Tomas 2209163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { 2210163a203dSDarrick J. Wong ext4_mb_unload_buddy(e4b); 2211163a203dSDarrick J. Wong return 0; 2212163a203dSDarrick J. Wong } 2213163a203dSDarrick J. Wong 2214c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 221515c006a2SRobin Dong max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, 2216c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 2217ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADFA11; /* debug value */ 2218c9de560dSAlex Tomas 2219c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 2220c9de560dSAlex Tomas ext4_fsblk_t start; 2221c9de560dSAlex Tomas 22225661bd68SAkinobu Mita start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + 22235661bd68SAkinobu Mita ex.fe_start; 2224c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 2225c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 2226c9de560dSAlex Tomas ac->ac_found++; 2227c9de560dSAlex Tomas ac->ac_b_ex = ex; 2228c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2229c9de560dSAlex Tomas } 2230c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 2231c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2232c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 2233c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 2234c9de560dSAlex Tomas ac->ac_found++; 2235c9de560dSAlex Tomas ac->ac_b_ex = ex; 2236c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2237c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 2238c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 2239c9de560dSAlex Tomas * number of blocks to an existing extent */ 2240c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 2241c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 2242c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 2243c9de560dSAlex Tomas ac->ac_found++; 2244c9de560dSAlex Tomas ac->ac_b_ex = ex; 2245c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2246c9de560dSAlex Tomas } 2247c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 2248e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 2249c9de560dSAlex Tomas 2250c9de560dSAlex Tomas return 0; 2251c9de560dSAlex Tomas } 2252c9de560dSAlex Tomas 2253c9de560dSAlex Tomas /* 2254c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 2255c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 2256c9de560dSAlex Tomas */ 2257089ceeccSEric Sandeen static noinline_for_stack 2258089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 2259c9de560dSAlex Tomas struct ext4_buddy *e4b) 2260c9de560dSAlex Tomas { 2261c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2262c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 2263c9de560dSAlex Tomas void *buddy; 2264c9de560dSAlex Tomas int i; 2265c9de560dSAlex Tomas int k; 2266c9de560dSAlex Tomas int max; 2267c9de560dSAlex Tomas 2268c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 22694b68f6dfSHarshad Shirwadkar for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) { 2270c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 2271c9de560dSAlex Tomas continue; 2272c9de560dSAlex Tomas 2273c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 2274c9de560dSAlex Tomas BUG_ON(buddy == NULL); 2275c9de560dSAlex Tomas 2276ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 2277eb576086SDmitry Monakhov if (k >= max) { 2278eb576086SDmitry Monakhov ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0, 2279eb576086SDmitry Monakhov "%d free clusters of order %d. But found 0", 2280eb576086SDmitry Monakhov grp->bb_counters[i], i); 2281eb576086SDmitry Monakhov ext4_mark_group_bitmap_corrupted(ac->ac_sb, 2282eb576086SDmitry Monakhov e4b->bd_group, 2283eb576086SDmitry Monakhov EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2284eb576086SDmitry Monakhov break; 2285eb576086SDmitry Monakhov } 2286c9de560dSAlex Tomas ac->ac_found++; 2287c9de560dSAlex Tomas 2288c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 2289c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 2290c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 2291c9de560dSAlex Tomas 2292c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2293c9de560dSAlex Tomas 229453f86b17SRitesh Harjani BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len); 2295c9de560dSAlex Tomas 2296c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 2297c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 2298c9de560dSAlex Tomas 2299c9de560dSAlex Tomas break; 2300c9de560dSAlex Tomas } 2301c9de560dSAlex Tomas } 2302c9de560dSAlex Tomas 2303c9de560dSAlex Tomas /* 2304c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 2305c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 2306c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 2307c9de560dSAlex Tomas */ 2308089ceeccSEric Sandeen static noinline_for_stack 2309089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 2310c9de560dSAlex Tomas struct ext4_buddy *e4b) 2311c9de560dSAlex Tomas { 2312c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2313c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2314c9de560dSAlex Tomas struct ext4_free_extent ex; 2315c9de560dSAlex Tomas int i; 2316c9de560dSAlex Tomas int free; 2317c9de560dSAlex Tomas 2318c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 2319907ea529STheodore Ts'o if (WARN_ON(free <= 0)) 2320907ea529STheodore Ts'o return; 2321c9de560dSAlex Tomas 2322c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 2323c9de560dSAlex Tomas 2324c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 2325ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 23267137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 23277137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 232826346ff6SAneesh Kumar K.V /* 2329e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 233026346ff6SAneesh Kumar K.V * free blocks even though group info says we 2331b483bb77SRandy Dunlap * have free blocks 233226346ff6SAneesh Kumar K.V */ 2333e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 233453accfa9STheodore Ts'o "%d free clusters as per " 2335fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 233626346ff6SAneesh Kumar K.V free); 2337736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2338736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2339c9de560dSAlex Tomas break; 2340c9de560dSAlex Tomas } 2341c9de560dSAlex Tomas 234215c006a2SRobin Dong mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); 2343907ea529STheodore Ts'o if (WARN_ON(ex.fe_len <= 0)) 2344907ea529STheodore Ts'o break; 234526346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 2346e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 234753accfa9STheodore Ts'o "%d free clusters as per " 2348fde4d95aSTheodore Ts'o "group info. But got %d blocks", 234926346ff6SAneesh Kumar K.V free, ex.fe_len); 2350736dedbbSWang Shilong ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group, 2351736dedbbSWang Shilong EXT4_GROUP_INFO_BBITMAP_CORRUPT); 2352e56eb659SAneesh Kumar K.V /* 2353e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 2354e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 2355e56eb659SAneesh Kumar K.V * without claiming the space. 2356e56eb659SAneesh Kumar K.V */ 2357e56eb659SAneesh Kumar K.V break; 235826346ff6SAneesh Kumar K.V } 2359ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADC0DE; /* debug value */ 2360c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 2361c9de560dSAlex Tomas 2362c9de560dSAlex Tomas i += ex.fe_len; 2363c9de560dSAlex Tomas free -= ex.fe_len; 2364c9de560dSAlex Tomas } 2365c9de560dSAlex Tomas 2366c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 2367c9de560dSAlex Tomas } 2368c9de560dSAlex Tomas 2369c9de560dSAlex Tomas /* 2370c9de560dSAlex Tomas * This is a special case for storages like raid5 2371506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 2372c9de560dSAlex Tomas */ 2373089ceeccSEric Sandeen static noinline_for_stack 2374089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 2375c9de560dSAlex Tomas struct ext4_buddy *e4b) 2376c9de560dSAlex Tomas { 2377c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2378c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2379c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 2380c9de560dSAlex Tomas struct ext4_free_extent ex; 2381c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 2382c9de560dSAlex Tomas ext4_fsblk_t a; 2383c9de560dSAlex Tomas ext4_grpblk_t i; 2384c9de560dSAlex Tomas int max; 2385c9de560dSAlex Tomas 2386c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 2387c9de560dSAlex Tomas 2388c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 23895661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 23905661bd68SAkinobu Mita 2391c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 2392c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 2393c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 2394c9de560dSAlex Tomas 23957137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 2396c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 239715c006a2SRobin Dong max = mb_find_extent(e4b, i, sbi->s_stripe, &ex); 2398c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 2399c9de560dSAlex Tomas ac->ac_found++; 2400ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADF00D; /* debug value */ 2401c9de560dSAlex Tomas ac->ac_b_ex = ex; 2402c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 2403c9de560dSAlex Tomas break; 2404c9de560dSAlex Tomas } 2405c9de560dSAlex Tomas } 2406c9de560dSAlex Tomas i += sbi->s_stripe; 2407c9de560dSAlex Tomas } 2408c9de560dSAlex Tomas } 2409c9de560dSAlex Tomas 241042ac1848SLukas Czerner /* 24118ef123feSRitesh Harjani * This is also called BEFORE we load the buddy bitmap. 241242ac1848SLukas Czerner * Returns either 1 or 0 indicating that the group is either suitable 24138ef123feSRitesh Harjani * for the allocation or not. 241442ac1848SLukas Czerner */ 24158ef123feSRitesh Harjani static bool ext4_mb_good_group(struct ext4_allocation_context *ac, 2416c9de560dSAlex Tomas ext4_group_t group, int cr) 2417c9de560dSAlex Tomas { 24188ef123feSRitesh Harjani ext4_grpblk_t free, fragments; 2419a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 2420c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 2421c9de560dSAlex Tomas 2422c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 24238a57d9d6SCurt Wohlgemuth 2424dddcd2f9Sbrookxu if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 24258ef123feSRitesh Harjani return false; 242601fc48e8STheodore Ts'o 2427dddcd2f9Sbrookxu free = grp->bb_free; 2428dddcd2f9Sbrookxu if (free == 0) 24298ef123feSRitesh Harjani return false; 2430c9de560dSAlex Tomas 2431c9de560dSAlex Tomas fragments = grp->bb_fragments; 2432c9de560dSAlex Tomas if (fragments == 0) 24338ef123feSRitesh Harjani return false; 2434c9de560dSAlex Tomas 2435c9de560dSAlex Tomas switch (cr) { 2436c9de560dSAlex Tomas case 0: 2437c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 2438c9de560dSAlex Tomas 2439a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 2440a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 2441a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 2442a4912123STheodore Ts'o ((group % flex_size) == 0)) 24438ef123feSRitesh Harjani return false; 2444a4912123STheodore Ts'o 2445dddcd2f9Sbrookxu if (free < ac->ac_g_ex.fe_len) 2446dddcd2f9Sbrookxu return false; 2447dddcd2f9Sbrookxu 24484b68f6dfSHarshad Shirwadkar if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb)) 24498ef123feSRitesh Harjani return true; 245040ae3487STheodore Ts'o 245140ae3487STheodore Ts'o if (grp->bb_largest_free_order < ac->ac_2order) 24528ef123feSRitesh Harjani return false; 245340ae3487STheodore Ts'o 24548ef123feSRitesh Harjani return true; 2455c9de560dSAlex Tomas case 1: 2456c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 24578ef123feSRitesh Harjani return true; 2458c9de560dSAlex Tomas break; 2459c9de560dSAlex Tomas case 2: 2460c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 24618ef123feSRitesh Harjani return true; 2462c9de560dSAlex Tomas break; 2463c9de560dSAlex Tomas case 3: 24648ef123feSRitesh Harjani return true; 2465c9de560dSAlex Tomas default: 2466c9de560dSAlex Tomas BUG(); 2467c9de560dSAlex Tomas } 2468c9de560dSAlex Tomas 24698ef123feSRitesh Harjani return false; 24708ef123feSRitesh Harjani } 24718ef123feSRitesh Harjani 24728ef123feSRitesh Harjani /* 24738ef123feSRitesh Harjani * This could return negative error code if something goes wrong 24748ef123feSRitesh Harjani * during ext4_mb_init_group(). This should not be called with 24758ef123feSRitesh Harjani * ext4_lock_group() held. 24768ef123feSRitesh Harjani */ 24778ef123feSRitesh Harjani static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac, 24788ef123feSRitesh Harjani ext4_group_t group, int cr) 24798ef123feSRitesh Harjani { 24808ef123feSRitesh Harjani struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 248199377830SRitesh Harjani struct super_block *sb = ac->ac_sb; 2482c1d2c7d4SAlex Zhuravlev struct ext4_sb_info *sbi = EXT4_SB(sb); 248399377830SRitesh Harjani bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK; 24848ef123feSRitesh Harjani ext4_grpblk_t free; 24858ef123feSRitesh Harjani int ret = 0; 24868ef123feSRitesh Harjani 2487a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats) 2488a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]); 248999377830SRitesh Harjani if (should_lock) 249099377830SRitesh Harjani ext4_lock_group(sb, group); 24918ef123feSRitesh Harjani free = grp->bb_free; 24928ef123feSRitesh Harjani if (free == 0) 24938ef123feSRitesh Harjani goto out; 24948ef123feSRitesh Harjani if (cr <= 2 && free < ac->ac_g_ex.fe_len) 24958ef123feSRitesh Harjani goto out; 24968ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 24978ef123feSRitesh Harjani goto out; 249899377830SRitesh Harjani if (should_lock) 249999377830SRitesh Harjani ext4_unlock_group(sb, group); 25008ef123feSRitesh Harjani 25018ef123feSRitesh Harjani /* We only do this if the grp has never been initialized */ 25028ef123feSRitesh Harjani if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 2503c1d2c7d4SAlex Zhuravlev struct ext4_group_desc *gdp = 2504c1d2c7d4SAlex Zhuravlev ext4_get_group_desc(sb, group, NULL); 2505c1d2c7d4SAlex Zhuravlev int ret; 2506c1d2c7d4SAlex Zhuravlev 2507c1d2c7d4SAlex Zhuravlev /* cr=0/1 is a very optimistic search to find large 2508c1d2c7d4SAlex Zhuravlev * good chunks almost for free. If buddy data is not 2509c1d2c7d4SAlex Zhuravlev * ready, then this optimization makes no sense. But 2510c1d2c7d4SAlex Zhuravlev * we never skip the first block group in a flex_bg, 2511c1d2c7d4SAlex Zhuravlev * since this gets used for metadata block allocation, 2512c1d2c7d4SAlex Zhuravlev * and we want to make sure we locate metadata blocks 2513c1d2c7d4SAlex Zhuravlev * in the first block group in the flex_bg if possible. 2514c1d2c7d4SAlex Zhuravlev */ 2515c1d2c7d4SAlex Zhuravlev if (cr < 2 && 2516c1d2c7d4SAlex Zhuravlev (!sbi->s_log_groups_per_flex || 2517c1d2c7d4SAlex Zhuravlev ((group & ((1 << sbi->s_log_groups_per_flex) - 1)) != 0)) && 2518c1d2c7d4SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2519c1d2c7d4SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) 2520c1d2c7d4SAlex Zhuravlev return 0; 2521c1d2c7d4SAlex Zhuravlev ret = ext4_mb_init_group(sb, group, GFP_NOFS); 25228ef123feSRitesh Harjani if (ret) 25238ef123feSRitesh Harjani return ret; 25248ef123feSRitesh Harjani } 25258ef123feSRitesh Harjani 252699377830SRitesh Harjani if (should_lock) 252799377830SRitesh Harjani ext4_lock_group(sb, group); 25288ef123feSRitesh Harjani ret = ext4_mb_good_group(ac, group, cr); 25298ef123feSRitesh Harjani out: 253099377830SRitesh Harjani if (should_lock) 253199377830SRitesh Harjani ext4_unlock_group(sb, group); 25328ef123feSRitesh Harjani return ret; 2533c9de560dSAlex Tomas } 2534c9de560dSAlex Tomas 2535cfd73237SAlex Zhuravlev /* 2536cfd73237SAlex Zhuravlev * Start prefetching @nr block bitmaps starting at @group. 2537cfd73237SAlex Zhuravlev * Return the next group which needs to be prefetched. 2538cfd73237SAlex Zhuravlev */ 25393d392b26STheodore Ts'o ext4_group_t ext4_mb_prefetch(struct super_block *sb, ext4_group_t group, 2540cfd73237SAlex Zhuravlev unsigned int nr, int *cnt) 2541cfd73237SAlex Zhuravlev { 2542cfd73237SAlex Zhuravlev ext4_group_t ngroups = ext4_get_groups_count(sb); 2543cfd73237SAlex Zhuravlev struct buffer_head *bh; 2544cfd73237SAlex Zhuravlev struct blk_plug plug; 2545cfd73237SAlex Zhuravlev 2546cfd73237SAlex Zhuravlev blk_start_plug(&plug); 2547cfd73237SAlex Zhuravlev while (nr-- > 0) { 2548cfd73237SAlex Zhuravlev struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, 2549cfd73237SAlex Zhuravlev NULL); 2550cfd73237SAlex Zhuravlev struct ext4_group_info *grp = ext4_get_group_info(sb, group); 2551cfd73237SAlex Zhuravlev 2552cfd73237SAlex Zhuravlev /* 2553cfd73237SAlex Zhuravlev * Prefetch block groups with free blocks; but don't 2554cfd73237SAlex Zhuravlev * bother if it is marked uninitialized on disk, since 2555cfd73237SAlex Zhuravlev * it won't require I/O to read. Also only try to 2556cfd73237SAlex Zhuravlev * prefetch once, so we avoid getblk() call, which can 2557cfd73237SAlex Zhuravlev * be expensive. 2558cfd73237SAlex Zhuravlev */ 2559cfd73237SAlex Zhuravlev if (!EXT4_MB_GRP_TEST_AND_SET_READ(grp) && 2560cfd73237SAlex Zhuravlev EXT4_MB_GRP_NEED_INIT(grp) && 2561cfd73237SAlex Zhuravlev ext4_free_group_clusters(sb, gdp) > 0 && 2562cfd73237SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2563cfd73237SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { 2564cfd73237SAlex Zhuravlev bh = ext4_read_block_bitmap_nowait(sb, group, true); 2565cfd73237SAlex Zhuravlev if (bh && !IS_ERR(bh)) { 2566cfd73237SAlex Zhuravlev if (!buffer_uptodate(bh) && cnt) 2567cfd73237SAlex Zhuravlev (*cnt)++; 2568cfd73237SAlex Zhuravlev brelse(bh); 2569cfd73237SAlex Zhuravlev } 2570cfd73237SAlex Zhuravlev } 2571cfd73237SAlex Zhuravlev if (++group >= ngroups) 2572cfd73237SAlex Zhuravlev group = 0; 2573cfd73237SAlex Zhuravlev } 2574cfd73237SAlex Zhuravlev blk_finish_plug(&plug); 2575cfd73237SAlex Zhuravlev return group; 2576cfd73237SAlex Zhuravlev } 2577cfd73237SAlex Zhuravlev 2578cfd73237SAlex Zhuravlev /* 2579cfd73237SAlex Zhuravlev * Prefetching reads the block bitmap into the buffer cache; but we 2580cfd73237SAlex Zhuravlev * need to make sure that the buddy bitmap in the page cache has been 2581cfd73237SAlex Zhuravlev * initialized. Note that ext4_mb_init_group() will block if the I/O 2582cfd73237SAlex Zhuravlev * is not yet completed, or indeed if it was not initiated by 2583cfd73237SAlex Zhuravlev * ext4_mb_prefetch did not start the I/O. 2584cfd73237SAlex Zhuravlev * 2585cfd73237SAlex Zhuravlev * TODO: We should actually kick off the buddy bitmap setup in a work 2586cfd73237SAlex Zhuravlev * queue when the buffer I/O is completed, so that we don't block 2587cfd73237SAlex Zhuravlev * waiting for the block allocation bitmap read to finish when 2588cfd73237SAlex Zhuravlev * ext4_mb_prefetch_fini is called from ext4_mb_regular_allocator(). 2589cfd73237SAlex Zhuravlev */ 25903d392b26STheodore Ts'o void ext4_mb_prefetch_fini(struct super_block *sb, ext4_group_t group, 2591cfd73237SAlex Zhuravlev unsigned int nr) 2592cfd73237SAlex Zhuravlev { 2593cfd73237SAlex Zhuravlev while (nr-- > 0) { 2594cfd73237SAlex Zhuravlev struct ext4_group_desc *gdp = ext4_get_group_desc(sb, group, 2595cfd73237SAlex Zhuravlev NULL); 2596cfd73237SAlex Zhuravlev struct ext4_group_info *grp = ext4_get_group_info(sb, group); 2597cfd73237SAlex Zhuravlev 2598cfd73237SAlex Zhuravlev if (!group) 2599cfd73237SAlex Zhuravlev group = ext4_get_groups_count(sb); 2600cfd73237SAlex Zhuravlev group--; 2601cfd73237SAlex Zhuravlev grp = ext4_get_group_info(sb, group); 2602cfd73237SAlex Zhuravlev 2603cfd73237SAlex Zhuravlev if (EXT4_MB_GRP_NEED_INIT(grp) && 2604cfd73237SAlex Zhuravlev ext4_free_group_clusters(sb, gdp) > 0 && 2605cfd73237SAlex Zhuravlev !(ext4_has_group_desc_csum(sb) && 2606cfd73237SAlex Zhuravlev (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)))) { 2607cfd73237SAlex Zhuravlev if (ext4_mb_init_group(sb, group, GFP_NOFS)) 2608cfd73237SAlex Zhuravlev break; 2609cfd73237SAlex Zhuravlev } 2610cfd73237SAlex Zhuravlev } 2611cfd73237SAlex Zhuravlev } 2612cfd73237SAlex Zhuravlev 26134ddfef7bSEric Sandeen static noinline_for_stack int 26144ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 2615c9de560dSAlex Tomas { 2616cfd73237SAlex Zhuravlev ext4_group_t prefetch_grp = 0, ngroups, group, i; 2617bbc4ec77SRitesh Harjani int cr = -1; 261842ac1848SLukas Czerner int err = 0, first_err = 0; 2619cfd73237SAlex Zhuravlev unsigned int nr = 0, prefetch_ios = 0; 2620c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2621c9de560dSAlex Tomas struct super_block *sb; 2622c9de560dSAlex Tomas struct ext4_buddy e4b; 262366d5e027Sbrookxu int lost; 2624c9de560dSAlex Tomas 2625c9de560dSAlex Tomas sb = ac->ac_sb; 2626c9de560dSAlex Tomas sbi = EXT4_SB(sb); 26278df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 2628fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 262912e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 2630fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 2631fb0a387dSEric Sandeen 2632c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2633c9de560dSAlex Tomas 2634c9de560dSAlex Tomas /* first, try the goal */ 2635c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 2636c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 2637c9de560dSAlex Tomas goto out; 2638c9de560dSAlex Tomas 2639c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2640c9de560dSAlex Tomas goto out; 2641c9de560dSAlex Tomas 2642c9de560dSAlex Tomas /* 2643e9a3cd48Sbrookxu * ac->ac_2order is set only if the fe_len is a power of 2 2644e9a3cd48Sbrookxu * if ac->ac_2order is set we also set criteria to 0 so that we 2645c9de560dSAlex Tomas * try exact allocation using buddy. 2646c9de560dSAlex Tomas */ 2647c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 2648c9de560dSAlex Tomas ac->ac_2order = 0; 2649c9de560dSAlex Tomas /* 2650c9de560dSAlex Tomas * We search using buddy data only if the order of the request 2651c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 2652b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 2653d9b22cf9SJan Kara * We also support searching for power-of-two requests only for 2654d9b22cf9SJan Kara * requests upto maximum buddy size we have constructed. 2655c9de560dSAlex Tomas */ 26564b68f6dfSHarshad Shirwadkar if (i >= sbi->s_mb_order2_reqs && i <= MB_NUM_ORDERS(sb)) { 2657c9de560dSAlex Tomas /* 2658c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2659c9de560dSAlex Tomas */ 2660c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 26611a5d5e5dSJeremy Cline ac->ac_2order = array_index_nospec(i - 1, 26624b68f6dfSHarshad Shirwadkar MB_NUM_ORDERS(sb)); 2663c9de560dSAlex Tomas } 2664c9de560dSAlex Tomas 26654ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 26664ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2667c9de560dSAlex Tomas /* TBD: may be hot point */ 2668c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2669c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2670c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2671c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2672c9de560dSAlex Tomas } 26734ba74d00STheodore Ts'o 2674c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2675c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2676c9de560dSAlex Tomas /* 2677c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2678c9de560dSAlex Tomas * cr == 3 try to get anything 2679c9de560dSAlex Tomas */ 2680c9de560dSAlex Tomas repeat: 2681c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2682c9de560dSAlex Tomas ac->ac_criteria = cr; 2683ed8f9c75SAneesh Kumar K.V /* 2684ed8f9c75SAneesh Kumar K.V * searching for the right group start 2685ed8f9c75SAneesh Kumar K.V * from the goal value specified 2686ed8f9c75SAneesh Kumar K.V */ 2687ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2688196e402aSHarshad Shirwadkar ac->ac_last_optimal_group = group; 2689196e402aSHarshad Shirwadkar ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups; 2690cfd73237SAlex Zhuravlev prefetch_grp = group; 2691ed8f9c75SAneesh Kumar K.V 2692196e402aSHarshad Shirwadkar for (i = 0; i < ngroups; group = next_linear_group(ac, group, ngroups), 2693196e402aSHarshad Shirwadkar i++) { 2694196e402aSHarshad Shirwadkar int ret = 0, new_cr; 2695196e402aSHarshad Shirwadkar 26962ed5724dSTheodore Ts'o cond_resched(); 2697196e402aSHarshad Shirwadkar 2698196e402aSHarshad Shirwadkar ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups); 2699196e402aSHarshad Shirwadkar if (new_cr != cr) { 2700196e402aSHarshad Shirwadkar cr = new_cr; 2701196e402aSHarshad Shirwadkar goto repeat; 2702196e402aSHarshad Shirwadkar } 2703c9de560dSAlex Tomas 2704cfd73237SAlex Zhuravlev /* 2705cfd73237SAlex Zhuravlev * Batch reads of the block allocation bitmaps 2706cfd73237SAlex Zhuravlev * to get multiple READs in flight; limit 2707cfd73237SAlex Zhuravlev * prefetching at cr=0/1, otherwise mballoc can 2708cfd73237SAlex Zhuravlev * spend a lot of time loading imperfect groups 2709cfd73237SAlex Zhuravlev */ 2710cfd73237SAlex Zhuravlev if ((prefetch_grp == group) && 2711cfd73237SAlex Zhuravlev (cr > 1 || 2712cfd73237SAlex Zhuravlev prefetch_ios < sbi->s_mb_prefetch_limit)) { 2713cfd73237SAlex Zhuravlev unsigned int curr_ios = prefetch_ios; 2714cfd73237SAlex Zhuravlev 2715cfd73237SAlex Zhuravlev nr = sbi->s_mb_prefetch; 2716cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 271782ef1370SChunguang Xu nr = 1 << sbi->s_log_groups_per_flex; 271882ef1370SChunguang Xu nr -= group & (nr - 1); 271982ef1370SChunguang Xu nr = min(nr, sbi->s_mb_prefetch); 2720cfd73237SAlex Zhuravlev } 2721cfd73237SAlex Zhuravlev prefetch_grp = ext4_mb_prefetch(sb, group, 2722cfd73237SAlex Zhuravlev nr, &prefetch_ios); 2723cfd73237SAlex Zhuravlev if (prefetch_ios == curr_ios) 2724cfd73237SAlex Zhuravlev nr = 0; 2725cfd73237SAlex Zhuravlev } 2726cfd73237SAlex Zhuravlev 27278a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 27288ef123feSRitesh Harjani ret = ext4_mb_good_group_nolock(ac, group, cr); 272942ac1848SLukas Czerner if (ret <= 0) { 273042ac1848SLukas Czerner if (!first_err) 273142ac1848SLukas Czerner first_err = ret; 2732c9de560dSAlex Tomas continue; 273342ac1848SLukas Czerner } 2734c9de560dSAlex Tomas 2735c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2736c9de560dSAlex Tomas if (err) 2737c9de560dSAlex Tomas goto out; 2738c9de560dSAlex Tomas 2739c9de560dSAlex Tomas ext4_lock_group(sb, group); 27408a57d9d6SCurt Wohlgemuth 27418a57d9d6SCurt Wohlgemuth /* 27428a57d9d6SCurt Wohlgemuth * We need to check again after locking the 27438a57d9d6SCurt Wohlgemuth * block group 27448a57d9d6SCurt Wohlgemuth */ 274542ac1848SLukas Czerner ret = ext4_mb_good_group(ac, group, cr); 27468ef123feSRitesh Harjani if (ret == 0) { 2747c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2748e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2749c9de560dSAlex Tomas continue; 2750c9de560dSAlex Tomas } 2751c9de560dSAlex Tomas 2752c9de560dSAlex Tomas ac->ac_groups_scanned++; 2753d9b22cf9SJan Kara if (cr == 0) 2754c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2755506bf2d8SEric Sandeen else if (cr == 1 && sbi->s_stripe && 2756506bf2d8SEric Sandeen !(ac->ac_g_ex.fe_len % sbi->s_stripe)) 2757c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2758c9de560dSAlex Tomas else 2759c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2760c9de560dSAlex Tomas 2761c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2762e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2763c9de560dSAlex Tomas 2764c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2765c9de560dSAlex Tomas break; 2766c9de560dSAlex Tomas } 2767a6c75eafSHarshad Shirwadkar /* Processed all groups and haven't found blocks */ 2768a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && i == ngroups) 2769a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_failed[cr]); 2770c9de560dSAlex Tomas } 2771c9de560dSAlex Tomas 2772c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2773c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2774c9de560dSAlex Tomas /* 2775c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2776c9de560dSAlex Tomas * the best chunk we've found so far 2777c9de560dSAlex Tomas */ 2778c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2779c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2780c9de560dSAlex Tomas /* 2781c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2782c9de560dSAlex Tomas * The only thing we can do is just take first 2783c9de560dSAlex Tomas * found block(s) 2784c9de560dSAlex Tomas */ 278566d5e027Sbrookxu lost = atomic_inc_return(&sbi->s_mb_lost_chunks); 278666d5e027Sbrookxu mb_debug(sb, "lost chunk, group: %u, start: %d, len: %d, lost: %d\n", 2787c55ee7d2Sbrookxu ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start, 2788c55ee7d2Sbrookxu ac->ac_b_ex.fe_len, lost); 2789c55ee7d2Sbrookxu 2790c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2791c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2792c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2793c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2794c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2795c9de560dSAlex Tomas cr = 3; 2796c9de560dSAlex Tomas goto repeat; 2797c9de560dSAlex Tomas } 2798c9de560dSAlex Tomas } 2799a6c75eafSHarshad Shirwadkar 2800a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND) 2801a6c75eafSHarshad Shirwadkar atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]); 2802c9de560dSAlex Tomas out: 280342ac1848SLukas Czerner if (!err && ac->ac_status != AC_STATUS_FOUND && first_err) 280442ac1848SLukas Czerner err = first_err; 2805bbc4ec77SRitesh Harjani 2806d3df1453SRitesh Harjani mb_debug(sb, "Best len %d, origin len %d, ac_status %u, ac_flags 0x%x, cr %d ret %d\n", 2807bbc4ec77SRitesh Harjani ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status, 2808bbc4ec77SRitesh Harjani ac->ac_flags, cr, err); 2809cfd73237SAlex Zhuravlev 2810cfd73237SAlex Zhuravlev if (nr) 2811cfd73237SAlex Zhuravlev ext4_mb_prefetch_fini(sb, prefetch_grp, nr); 2812cfd73237SAlex Zhuravlev 2813c9de560dSAlex Tomas return err; 2814c9de560dSAlex Tomas } 2815c9de560dSAlex Tomas 2816c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2817c9de560dSAlex Tomas { 2818247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2819c9de560dSAlex Tomas ext4_group_t group; 2820c9de560dSAlex Tomas 28218df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2822c9de560dSAlex Tomas return NULL; 2823c9de560dSAlex Tomas group = *pos + 1; 2824a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2825c9de560dSAlex Tomas } 2826c9de560dSAlex Tomas 2827c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2828c9de560dSAlex Tomas { 2829247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2830c9de560dSAlex Tomas ext4_group_t group; 2831c9de560dSAlex Tomas 2832c9de560dSAlex Tomas ++*pos; 28338df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2834c9de560dSAlex Tomas return NULL; 2835c9de560dSAlex Tomas group = *pos + 1; 2836a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2837c9de560dSAlex Tomas } 2838c9de560dSAlex Tomas 2839c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2840c9de560dSAlex Tomas { 2841247dbed8SChristoph Hellwig struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2842a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2843c9de560dSAlex Tomas int i; 28441c8457caSAditya Kali int err, buddy_loaded = 0; 2845c9de560dSAlex Tomas struct ext4_buddy e4b; 28461c8457caSAditya Kali struct ext4_group_info *grinfo; 28472df2c340SArnd Bergmann unsigned char blocksize_bits = min_t(unsigned char, 28482df2c340SArnd Bergmann sb->s_blocksize_bits, 28492df2c340SArnd Bergmann EXT4_MAX_BLOCK_LOG_SIZE); 2850c9de560dSAlex Tomas struct sg { 2851c9de560dSAlex Tomas struct ext4_group_info info; 2852b80b32b6STheodore Ts'o ext4_grpblk_t counters[EXT4_MAX_BLOCK_LOG_SIZE + 2]; 2853c9de560dSAlex Tomas } sg; 2854c9de560dSAlex Tomas 2855c9de560dSAlex Tomas group--; 2856c9de560dSAlex Tomas if (group == 0) 285797b4af2fSRasmus Villemoes seq_puts(seq, "#group: free frags first [" 285897b4af2fSRasmus Villemoes " 2^0 2^1 2^2 2^3 2^4 2^5 2^6 " 2859802cf1f9SHuaitong Han " 2^7 2^8 2^9 2^10 2^11 2^12 2^13 ]\n"); 2860c9de560dSAlex Tomas 2861b80b32b6STheodore Ts'o i = (blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2862b80b32b6STheodore Ts'o sizeof(struct ext4_group_info); 2863b80b32b6STheodore Ts'o 28641c8457caSAditya Kali grinfo = ext4_get_group_info(sb, group); 28651c8457caSAditya Kali /* Load the group info in memory only if not already loaded. */ 28661c8457caSAditya Kali if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { 2867c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2868c9de560dSAlex Tomas if (err) { 2869a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2870c9de560dSAlex Tomas return 0; 2871c9de560dSAlex Tomas } 28721c8457caSAditya Kali buddy_loaded = 1; 28731c8457caSAditya Kali } 28741c8457caSAditya Kali 2875b80b32b6STheodore Ts'o memcpy(&sg, ext4_get_group_info(sb, group), i); 28761c8457caSAditya Kali 28771c8457caSAditya Kali if (buddy_loaded) 2878e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2879c9de560dSAlex Tomas 2880a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2881c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2882c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 28832df2c340SArnd Bergmann seq_printf(seq, " %-5u", i <= blocksize_bits + 1 ? 2884c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2885e0d438c7SXu Wang seq_puts(seq, " ]\n"); 2886c9de560dSAlex Tomas 2887c9de560dSAlex Tomas return 0; 2888c9de560dSAlex Tomas } 2889c9de560dSAlex Tomas 2890c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2891c9de560dSAlex Tomas { 2892c9de560dSAlex Tomas } 2893c9de560dSAlex Tomas 2894247dbed8SChristoph Hellwig const struct seq_operations ext4_mb_seq_groups_ops = { 2895c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2896c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2897c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2898c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2899c9de560dSAlex Tomas }; 2900c9de560dSAlex Tomas 2901a6c75eafSHarshad Shirwadkar int ext4_seq_mb_stats_show(struct seq_file *seq, void *offset) 2902a6c75eafSHarshad Shirwadkar { 2903a6c75eafSHarshad Shirwadkar struct super_block *sb = (struct super_block *)seq->private; 2904a6c75eafSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 2905a6c75eafSHarshad Shirwadkar 2906a6c75eafSHarshad Shirwadkar seq_puts(seq, "mballoc:\n"); 2907a6c75eafSHarshad Shirwadkar if (!sbi->s_mb_stats) { 2908a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tmb stats collection turned off.\n"); 2909a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tTo enable, please write \"1\" to sysfs file mb_stats.\n"); 2910a6c75eafSHarshad Shirwadkar return 0; 2911a6c75eafSHarshad Shirwadkar } 2912a6c75eafSHarshad Shirwadkar seq_printf(seq, "\treqs: %u\n", atomic_read(&sbi->s_bal_reqs)); 2913a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tsuccess: %u\n", atomic_read(&sbi->s_bal_success)); 2914a6c75eafSHarshad Shirwadkar 2915a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tgroups_scanned: %u\n", atomic_read(&sbi->s_bal_groups_scanned)); 2916a6c75eafSHarshad Shirwadkar 2917a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tcr0_stats:\n"); 2918a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[0])); 2919a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 2920a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_groups_considered[0])); 2921a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 2922a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_failed[0])); 2923196e402aSHarshad Shirwadkar seq_printf(seq, "\t\tbad_suggestions: %u\n", 2924196e402aSHarshad Shirwadkar atomic_read(&sbi->s_bal_cr0_bad_suggestions)); 2925a6c75eafSHarshad Shirwadkar 2926a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tcr1_stats:\n"); 2927a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[1])); 2928a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 2929a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_groups_considered[1])); 2930a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 2931a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_failed[1])); 2932196e402aSHarshad Shirwadkar seq_printf(seq, "\t\tbad_suggestions: %u\n", 2933196e402aSHarshad Shirwadkar atomic_read(&sbi->s_bal_cr1_bad_suggestions)); 2934a6c75eafSHarshad Shirwadkar 2935a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tcr2_stats:\n"); 2936a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[2])); 2937a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 2938a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_groups_considered[2])); 2939a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 2940a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_failed[2])); 2941a6c75eafSHarshad Shirwadkar 2942a6c75eafSHarshad Shirwadkar seq_puts(seq, "\tcr3_stats:\n"); 2943a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\thits: %llu\n", atomic64_read(&sbi->s_bal_cX_hits[3])); 2944a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgroups_considered: %llu\n", 2945a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_groups_considered[3])); 2946a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tuseless_loops: %llu\n", 2947a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_bal_cX_failed[3])); 2948a6c75eafSHarshad Shirwadkar seq_printf(seq, "\textents_scanned: %u\n", atomic_read(&sbi->s_bal_ex_scanned)); 2949a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tgoal_hits: %u\n", atomic_read(&sbi->s_bal_goals)); 2950a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\t2^n_hits: %u\n", atomic_read(&sbi->s_bal_2orders)); 2951a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tbreaks: %u\n", atomic_read(&sbi->s_bal_breaks)); 2952a6c75eafSHarshad Shirwadkar seq_printf(seq, "\t\tlost: %u\n", atomic_read(&sbi->s_mb_lost_chunks)); 2953a6c75eafSHarshad Shirwadkar 2954a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tbuddies_generated: %u/%u\n", 2955a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_mb_buddies_generated), 2956a6c75eafSHarshad Shirwadkar ext4_get_groups_count(sb)); 2957a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tbuddies_time_used: %llu\n", 2958a6c75eafSHarshad Shirwadkar atomic64_read(&sbi->s_mb_generation_time)); 2959a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tpreallocated: %u\n", 2960a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_mb_preallocated)); 2961a6c75eafSHarshad Shirwadkar seq_printf(seq, "\tdiscarded: %u\n", 2962a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_mb_discarded)); 2963a6c75eafSHarshad Shirwadkar return 0; 2964a6c75eafSHarshad Shirwadkar } 2965a6c75eafSHarshad Shirwadkar 2966*f68f4063SHarshad Shirwadkar static void *ext4_mb_seq_structs_summary_start(struct seq_file *seq, loff_t *pos) 2967*f68f4063SHarshad Shirwadkar { 2968*f68f4063SHarshad Shirwadkar struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2969*f68f4063SHarshad Shirwadkar unsigned long position; 2970*f68f4063SHarshad Shirwadkar 2971*f68f4063SHarshad Shirwadkar read_lock(&EXT4_SB(sb)->s_mb_rb_lock); 2972*f68f4063SHarshad Shirwadkar 2973*f68f4063SHarshad Shirwadkar if (*pos < 0 || *pos >= MB_NUM_ORDERS(sb) + 1) 2974*f68f4063SHarshad Shirwadkar return NULL; 2975*f68f4063SHarshad Shirwadkar position = *pos + 1; 2976*f68f4063SHarshad Shirwadkar return (void *) ((unsigned long) position); 2977*f68f4063SHarshad Shirwadkar } 2978*f68f4063SHarshad Shirwadkar 2979*f68f4063SHarshad Shirwadkar static void *ext4_mb_seq_structs_summary_next(struct seq_file *seq, void *v, loff_t *pos) 2980*f68f4063SHarshad Shirwadkar { 2981*f68f4063SHarshad Shirwadkar struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2982*f68f4063SHarshad Shirwadkar unsigned long position; 2983*f68f4063SHarshad Shirwadkar 2984*f68f4063SHarshad Shirwadkar ++*pos; 2985*f68f4063SHarshad Shirwadkar if (*pos < 0 || *pos >= MB_NUM_ORDERS(sb) + 1) 2986*f68f4063SHarshad Shirwadkar return NULL; 2987*f68f4063SHarshad Shirwadkar position = *pos + 1; 2988*f68f4063SHarshad Shirwadkar return (void *) ((unsigned long) position); 2989*f68f4063SHarshad Shirwadkar } 2990*f68f4063SHarshad Shirwadkar 2991*f68f4063SHarshad Shirwadkar static int ext4_mb_seq_structs_summary_show(struct seq_file *seq, void *v) 2992*f68f4063SHarshad Shirwadkar { 2993*f68f4063SHarshad Shirwadkar struct super_block *sb = PDE_DATA(file_inode(seq->file)); 2994*f68f4063SHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 2995*f68f4063SHarshad Shirwadkar unsigned long position = ((unsigned long) v); 2996*f68f4063SHarshad Shirwadkar struct ext4_group_info *grp; 2997*f68f4063SHarshad Shirwadkar struct rb_node *n; 2998*f68f4063SHarshad Shirwadkar unsigned int count, min, max; 2999*f68f4063SHarshad Shirwadkar 3000*f68f4063SHarshad Shirwadkar position--; 3001*f68f4063SHarshad Shirwadkar if (position >= MB_NUM_ORDERS(sb)) { 3002*f68f4063SHarshad Shirwadkar seq_puts(seq, "fragment_size_tree:\n"); 3003*f68f4063SHarshad Shirwadkar n = rb_first(&sbi->s_mb_avg_fragment_size_root); 3004*f68f4063SHarshad Shirwadkar if (!n) { 3005*f68f4063SHarshad Shirwadkar seq_puts(seq, "\ttree_min: 0\n\ttree_max: 0\n\ttree_nodes: 0\n"); 3006*f68f4063SHarshad Shirwadkar return 0; 3007*f68f4063SHarshad Shirwadkar } 3008*f68f4063SHarshad Shirwadkar grp = rb_entry(n, struct ext4_group_info, bb_avg_fragment_size_rb); 3009*f68f4063SHarshad Shirwadkar min = grp->bb_fragments ? grp->bb_free / grp->bb_fragments : 0; 3010*f68f4063SHarshad Shirwadkar count = 1; 3011*f68f4063SHarshad Shirwadkar while (rb_next(n)) { 3012*f68f4063SHarshad Shirwadkar count++; 3013*f68f4063SHarshad Shirwadkar n = rb_next(n); 3014*f68f4063SHarshad Shirwadkar } 3015*f68f4063SHarshad Shirwadkar grp = rb_entry(n, struct ext4_group_info, bb_avg_fragment_size_rb); 3016*f68f4063SHarshad Shirwadkar max = grp->bb_fragments ? grp->bb_free / grp->bb_fragments : 0; 3017*f68f4063SHarshad Shirwadkar 3018*f68f4063SHarshad Shirwadkar seq_printf(seq, "\ttree_min: %u\n\ttree_max: %u\n\ttree_nodes: %u\n", 3019*f68f4063SHarshad Shirwadkar min, max, count); 3020*f68f4063SHarshad Shirwadkar return 0; 3021*f68f4063SHarshad Shirwadkar } 3022*f68f4063SHarshad Shirwadkar 3023*f68f4063SHarshad Shirwadkar if (position == 0) { 3024*f68f4063SHarshad Shirwadkar seq_printf(seq, "optimize_scan: %d\n", 3025*f68f4063SHarshad Shirwadkar test_opt2(sb, MB_OPTIMIZE_SCAN) ? 1 : 0); 3026*f68f4063SHarshad Shirwadkar seq_puts(seq, "max_free_order_lists:\n"); 3027*f68f4063SHarshad Shirwadkar } 3028*f68f4063SHarshad Shirwadkar count = 0; 3029*f68f4063SHarshad Shirwadkar list_for_each_entry(grp, &sbi->s_mb_largest_free_orders[position], 3030*f68f4063SHarshad Shirwadkar bb_largest_free_order_node) 3031*f68f4063SHarshad Shirwadkar count++; 3032*f68f4063SHarshad Shirwadkar seq_printf(seq, "\tlist_order_%u_groups: %u\n", 3033*f68f4063SHarshad Shirwadkar (unsigned int)position, count); 3034*f68f4063SHarshad Shirwadkar 3035*f68f4063SHarshad Shirwadkar return 0; 3036*f68f4063SHarshad Shirwadkar } 3037*f68f4063SHarshad Shirwadkar 3038*f68f4063SHarshad Shirwadkar static void ext4_mb_seq_structs_summary_stop(struct seq_file *seq, void *v) 3039*f68f4063SHarshad Shirwadkar { 3040*f68f4063SHarshad Shirwadkar struct super_block *sb = PDE_DATA(file_inode(seq->file)); 3041*f68f4063SHarshad Shirwadkar 3042*f68f4063SHarshad Shirwadkar read_unlock(&EXT4_SB(sb)->s_mb_rb_lock); 3043*f68f4063SHarshad Shirwadkar } 3044*f68f4063SHarshad Shirwadkar 3045*f68f4063SHarshad Shirwadkar const struct seq_operations ext4_mb_seq_structs_summary_ops = { 3046*f68f4063SHarshad Shirwadkar .start = ext4_mb_seq_structs_summary_start, 3047*f68f4063SHarshad Shirwadkar .next = ext4_mb_seq_structs_summary_next, 3048*f68f4063SHarshad Shirwadkar .stop = ext4_mb_seq_structs_summary_stop, 3049*f68f4063SHarshad Shirwadkar .show = ext4_mb_seq_structs_summary_show, 3050*f68f4063SHarshad Shirwadkar }; 3051*f68f4063SHarshad Shirwadkar 3052fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 3053fb1813f4SCurt Wohlgemuth { 3054fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 3055fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 3056fb1813f4SCurt Wohlgemuth 3057fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 3058fb1813f4SCurt Wohlgemuth return cachep; 3059fb1813f4SCurt Wohlgemuth } 30605f21b0e6SFrederic Bohe 306128623c2fSTheodore Ts'o /* 306228623c2fSTheodore Ts'o * Allocate the top-level s_group_info array for the specified number 306328623c2fSTheodore Ts'o * of groups 306428623c2fSTheodore Ts'o */ 306528623c2fSTheodore Ts'o int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) 306628623c2fSTheodore Ts'o { 306728623c2fSTheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 306828623c2fSTheodore Ts'o unsigned size; 3069df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***old_groupinfo, ***new_groupinfo; 307028623c2fSTheodore Ts'o 307128623c2fSTheodore Ts'o size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> 307228623c2fSTheodore Ts'o EXT4_DESC_PER_BLOCK_BITS(sb); 307328623c2fSTheodore Ts'o if (size <= sbi->s_group_info_size) 307428623c2fSTheodore Ts'o return 0; 307528623c2fSTheodore Ts'o 307628623c2fSTheodore Ts'o size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); 3077a7c3e901SMichal Hocko new_groupinfo = kvzalloc(size, GFP_KERNEL); 307828623c2fSTheodore Ts'o if (!new_groupinfo) { 307928623c2fSTheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 308028623c2fSTheodore Ts'o return -ENOMEM; 308128623c2fSTheodore Ts'o } 3082df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3083df3da4eaSSuraj Jitindar Singh old_groupinfo = rcu_dereference(sbi->s_group_info); 3084df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 3085df3da4eaSSuraj Jitindar Singh memcpy(new_groupinfo, old_groupinfo, 308628623c2fSTheodore Ts'o sbi->s_group_info_size * sizeof(*sbi->s_group_info)); 3087df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3088df3da4eaSSuraj Jitindar Singh rcu_assign_pointer(sbi->s_group_info, new_groupinfo); 308928623c2fSTheodore Ts'o sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); 3090df3da4eaSSuraj Jitindar Singh if (old_groupinfo) 3091df3da4eaSSuraj Jitindar Singh ext4_kvfree_array_rcu(old_groupinfo); 309228623c2fSTheodore Ts'o ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", 309328623c2fSTheodore Ts'o sbi->s_group_info_size); 309428623c2fSTheodore Ts'o return 0; 309528623c2fSTheodore Ts'o } 309628623c2fSTheodore Ts'o 30975f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 3098920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 30995f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 31005f21b0e6SFrederic Bohe { 3101fb1813f4SCurt Wohlgemuth int i; 31025f21b0e6SFrederic Bohe int metalen = 0; 3103df3da4eaSSuraj Jitindar Singh int idx = group >> EXT4_DESC_PER_BLOCK_BITS(sb); 31045f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 31055f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 3106fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 31075f21b0e6SFrederic Bohe 31085f21b0e6SFrederic Bohe /* 31095f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 31105f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 31115f21b0e6SFrederic Bohe * to ext4_group_info structures 31125f21b0e6SFrederic Bohe */ 31135f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 31145f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 31155f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 31164fdb5543SDmitry Monakhov meta_group_info = kmalloc(metalen, GFP_NOFS); 31175f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 31187f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate mem " 31199d8b9ec4STheodore Ts'o "for a buddy group"); 31205f21b0e6SFrederic Bohe goto exit_meta_group_info; 31215f21b0e6SFrederic Bohe } 3122df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3123df3da4eaSSuraj Jitindar Singh rcu_dereference(sbi->s_group_info)[idx] = meta_group_info; 3124df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 31255f21b0e6SFrederic Bohe } 31265f21b0e6SFrederic Bohe 3127df3da4eaSSuraj Jitindar Singh meta_group_info = sbi_array_rcu_deref(sbi, s_group_info, idx); 31285f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 31295f21b0e6SFrederic Bohe 31304fdb5543SDmitry Monakhov meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_NOFS); 31315f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 31327f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); 31335f21b0e6SFrederic Bohe goto exit_group_info; 31345f21b0e6SFrederic Bohe } 31355f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 31365f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 31375f21b0e6SFrederic Bohe 31385f21b0e6SFrederic Bohe /* 31395f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 31405f21b0e6SFrederic Bohe * empty groups without initialization 31415f21b0e6SFrederic Bohe */ 31428844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 31438844618dSTheodore Ts'o (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 31445f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 3145cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, group, desc); 31465f21b0e6SFrederic Bohe } else { 31475f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 3148021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, desc); 31495f21b0e6SFrederic Bohe } 31505f21b0e6SFrederic Bohe 31515f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 3152920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 315364e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 3154196e402aSHarshad Shirwadkar INIT_LIST_HEAD(&meta_group_info[i]->bb_largest_free_order_node); 3155196e402aSHarshad Shirwadkar RB_CLEAR_NODE(&meta_group_info[i]->bb_avg_fragment_size_rb); 31568a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 3157196e402aSHarshad Shirwadkar meta_group_info[i]->bb_group = group; 31585f21b0e6SFrederic Bohe 3159a3450215SRitesh Harjani mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group); 31605f21b0e6SFrederic Bohe return 0; 31615f21b0e6SFrederic Bohe 31625f21b0e6SFrederic Bohe exit_group_info: 31635f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 3164caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 3165df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 3166df3da4eaSSuraj Jitindar Singh 3167df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3168df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3169df3da4eaSSuraj Jitindar Singh kfree(group_info[idx]); 3170df3da4eaSSuraj Jitindar Singh group_info[idx] = NULL; 3171df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3172caaf7a29STao Ma } 31735f21b0e6SFrederic Bohe exit_meta_group_info: 31745f21b0e6SFrederic Bohe return -ENOMEM; 31755f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 31765f21b0e6SFrederic Bohe 3177c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 3178c9de560dSAlex Tomas { 31798df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 3180c9de560dSAlex Tomas ext4_group_t i; 3181c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 318228623c2fSTheodore Ts'o int err; 31835f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 3184df3da4eaSSuraj Jitindar Singh struct ext4_group_info ***group_info; 3185fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 3186c9de560dSAlex Tomas 318728623c2fSTheodore Ts'o err = ext4_mb_alloc_groupinfo(sb, ngroups); 318828623c2fSTheodore Ts'o if (err) 318928623c2fSTheodore Ts'o return err; 31905f21b0e6SFrederic Bohe 3191c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 3192c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 31939d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 3194c9de560dSAlex Tomas goto err_freesgi; 3195c9de560dSAlex Tomas } 319648e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 319748e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 319848e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 319948e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 320048e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 3201c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 32028df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 32034b99faa2SKhazhismel Kumykov cond_resched(); 3204c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 3205c9de560dSAlex Tomas if (desc == NULL) { 32069d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 3207c9de560dSAlex Tomas goto err_freebuddy; 3208c9de560dSAlex Tomas } 32095f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 32105f21b0e6SFrederic Bohe goto err_freebuddy; 3211c9de560dSAlex Tomas } 3212c9de560dSAlex Tomas 3213cfd73237SAlex Zhuravlev if (ext4_has_feature_flex_bg(sb)) { 3214f91436d5SSabyrzhan Tasbolatov /* a single flex group is supposed to be read by a single IO. 3215f91436d5SSabyrzhan Tasbolatov * 2 ^ s_log_groups_per_flex != UINT_MAX as s_mb_prefetch is 3216f91436d5SSabyrzhan Tasbolatov * unsigned integer, so the maximum shift is 32. 3217f91436d5SSabyrzhan Tasbolatov */ 3218f91436d5SSabyrzhan Tasbolatov if (sbi->s_es->s_log_groups_per_flex >= 32) { 3219f91436d5SSabyrzhan Tasbolatov ext4_msg(sb, KERN_ERR, "too many log groups per flexible block group"); 3220f91436d5SSabyrzhan Tasbolatov goto err_freesgi; 3221f91436d5SSabyrzhan Tasbolatov } 3222f91436d5SSabyrzhan Tasbolatov sbi->s_mb_prefetch = min_t(uint, 1 << sbi->s_es->s_log_groups_per_flex, 322382ef1370SChunguang Xu BLK_MAX_SEGMENT_SIZE >> (sb->s_blocksize_bits - 9)); 3224cfd73237SAlex Zhuravlev sbi->s_mb_prefetch *= 8; /* 8 prefetch IOs in flight at most */ 3225cfd73237SAlex Zhuravlev } else { 3226cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = 32; 3227cfd73237SAlex Zhuravlev } 3228cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch > ext4_get_groups_count(sb)) 3229cfd73237SAlex Zhuravlev sbi->s_mb_prefetch = ext4_get_groups_count(sb); 3230cfd73237SAlex Zhuravlev /* now many real IOs to prefetch within a single allocation at cr=0 3231cfd73237SAlex Zhuravlev * given cr=0 is an CPU-related optimization we shouldn't try to 3232cfd73237SAlex Zhuravlev * load too many groups, at some point we should start to use what 3233cfd73237SAlex Zhuravlev * we've got in memory. 3234cfd73237SAlex Zhuravlev * with an average random access time 5ms, it'd take a second to get 3235cfd73237SAlex Zhuravlev * 200 groups (* N with flex_bg), so let's make this limit 4 3236cfd73237SAlex Zhuravlev */ 3237cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = sbi->s_mb_prefetch * 4; 3238cfd73237SAlex Zhuravlev if (sbi->s_mb_prefetch_limit > ext4_get_groups_count(sb)) 3239cfd73237SAlex Zhuravlev sbi->s_mb_prefetch_limit = ext4_get_groups_count(sb); 3240cfd73237SAlex Zhuravlev 3241c9de560dSAlex Tomas return 0; 3242c9de560dSAlex Tomas 3243c9de560dSAlex Tomas err_freebuddy: 3244fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 3245f1fa3342SRoel Kluin while (i-- > 0) 3246fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, ext4_get_group_info(sb, i)); 324728623c2fSTheodore Ts'o i = sbi->s_group_info_size; 3248df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3249df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3250f1fa3342SRoel Kluin while (i-- > 0) 3251df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 3252df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3253c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 3254c9de560dSAlex Tomas err_freesgi: 3255df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3256df3da4eaSSuraj Jitindar Singh kvfree(rcu_dereference(sbi->s_group_info)); 3257df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3258c9de560dSAlex Tomas return -ENOMEM; 3259c9de560dSAlex Tomas } 3260c9de560dSAlex Tomas 32612892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 32622892c15dSEric Sandeen { 32632892c15dSEric Sandeen int i; 32642892c15dSEric Sandeen 32652892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 32662892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 32672892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 32682892c15dSEric Sandeen } 32692892c15dSEric Sandeen } 32702892c15dSEric Sandeen 32712892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 32722892c15dSEric Sandeen { 32732892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 32742892c15dSEric Sandeen int slab_size; 32752892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 32762892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 32772892c15dSEric Sandeen struct kmem_cache *cachep; 32782892c15dSEric Sandeen 32792892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 32802892c15dSEric Sandeen return -EINVAL; 32812892c15dSEric Sandeen 32822892c15dSEric Sandeen if (unlikely(cache_index < 0)) 32832892c15dSEric Sandeen cache_index = 0; 32842892c15dSEric Sandeen 32852892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 32862892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 32872892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 32882892c15dSEric Sandeen return 0; /* Already created */ 32892892c15dSEric Sandeen } 32902892c15dSEric Sandeen 32912892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 32922892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 32932892c15dSEric Sandeen 32942892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 32952892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 32962892c15dSEric Sandeen NULL); 32972892c15dSEric Sandeen 3298823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 3299823ba01fSTao Ma 33002892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 33012892c15dSEric Sandeen if (!cachep) { 33029d8b9ec4STheodore Ts'o printk(KERN_EMERG 33039d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 33042892c15dSEric Sandeen return -ENOMEM; 33052892c15dSEric Sandeen } 33062892c15dSEric Sandeen 33072892c15dSEric Sandeen return 0; 33082892c15dSEric Sandeen } 33092892c15dSEric Sandeen 33109d99012fSAkira Fujita int ext4_mb_init(struct super_block *sb) 3311c9de560dSAlex Tomas { 3312c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 33136be2ded1SAneesh Kumar K.V unsigned i, j; 3314935244cdSNicolai Stange unsigned offset, offset_incr; 3315c9de560dSAlex Tomas unsigned max; 331674767c5aSShen Feng int ret; 3317c9de560dSAlex Tomas 33184b68f6dfSHarshad Shirwadkar i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_offsets); 3319c9de560dSAlex Tomas 3320c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 3321c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 3322fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3323fb1813f4SCurt Wohlgemuth goto out; 3324c9de560dSAlex Tomas } 3325ff7ef329SYasunori Goto 33264b68f6dfSHarshad Shirwadkar i = MB_NUM_ORDERS(sb) * sizeof(*sbi->s_mb_maxs); 3327c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 3328c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 3329fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3330fb1813f4SCurt Wohlgemuth goto out; 3331fb1813f4SCurt Wohlgemuth } 3332fb1813f4SCurt Wohlgemuth 33332892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 33342892c15dSEric Sandeen if (ret < 0) 3335fb1813f4SCurt Wohlgemuth goto out; 3336c9de560dSAlex Tomas 3337c9de560dSAlex Tomas /* order 0 is regular bitmap */ 3338c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 3339c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 3340c9de560dSAlex Tomas 3341c9de560dSAlex Tomas i = 1; 3342c9de560dSAlex Tomas offset = 0; 3343935244cdSNicolai Stange offset_incr = 1 << (sb->s_blocksize_bits - 1); 3344c9de560dSAlex Tomas max = sb->s_blocksize << 2; 3345c9de560dSAlex Tomas do { 3346c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 3347c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 3348935244cdSNicolai Stange offset += offset_incr; 3349935244cdSNicolai Stange offset_incr = offset_incr >> 1; 3350c9de560dSAlex Tomas max = max >> 1; 3351c9de560dSAlex Tomas i++; 33524b68f6dfSHarshad Shirwadkar } while (i < MB_NUM_ORDERS(sb)); 33534b68f6dfSHarshad Shirwadkar 3354196e402aSHarshad Shirwadkar sbi->s_mb_avg_fragment_size_root = RB_ROOT; 3355196e402aSHarshad Shirwadkar sbi->s_mb_largest_free_orders = 3356196e402aSHarshad Shirwadkar kmalloc_array(MB_NUM_ORDERS(sb), sizeof(struct list_head), 3357196e402aSHarshad Shirwadkar GFP_KERNEL); 3358196e402aSHarshad Shirwadkar if (!sbi->s_mb_largest_free_orders) { 3359196e402aSHarshad Shirwadkar ret = -ENOMEM; 3360196e402aSHarshad Shirwadkar goto out; 3361196e402aSHarshad Shirwadkar } 3362196e402aSHarshad Shirwadkar sbi->s_mb_largest_free_orders_locks = 3363196e402aSHarshad Shirwadkar kmalloc_array(MB_NUM_ORDERS(sb), sizeof(rwlock_t), 3364196e402aSHarshad Shirwadkar GFP_KERNEL); 3365196e402aSHarshad Shirwadkar if (!sbi->s_mb_largest_free_orders_locks) { 3366196e402aSHarshad Shirwadkar ret = -ENOMEM; 3367196e402aSHarshad Shirwadkar goto out; 3368196e402aSHarshad Shirwadkar } 3369196e402aSHarshad Shirwadkar for (i = 0; i < MB_NUM_ORDERS(sb); i++) { 3370196e402aSHarshad Shirwadkar INIT_LIST_HEAD(&sbi->s_mb_largest_free_orders[i]); 3371196e402aSHarshad Shirwadkar rwlock_init(&sbi->s_mb_largest_free_orders_locks[i]); 3372196e402aSHarshad Shirwadkar } 3373196e402aSHarshad Shirwadkar rwlock_init(&sbi->s_mb_rb_lock); 3374c9de560dSAlex Tomas 3375c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 3376d08854f5STheodore Ts'o sbi->s_mb_free_pending = 0; 3377a0154344SDaeho Jeong INIT_LIST_HEAD(&sbi->s_freed_data_list); 3378c9de560dSAlex Tomas 3379c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 3380c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 3381c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 3382c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 3383c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 338427bc446eSbrookxu sbi->s_mb_max_inode_prealloc = MB_DEFAULT_MAX_INODE_PREALLOC; 338527baebb8STheodore Ts'o /* 338627baebb8STheodore Ts'o * The default group preallocation is 512, which for 4k block 338727baebb8STheodore Ts'o * sizes translates to 2 megabytes. However for bigalloc file 338827baebb8STheodore Ts'o * systems, this is probably too big (i.e, if the cluster size 338927baebb8STheodore Ts'o * is 1 megabyte, then group preallocation size becomes half a 339027baebb8STheodore Ts'o * gigabyte!). As a default, we will keep a two megabyte 339127baebb8STheodore Ts'o * group pralloc size for cluster sizes up to 64k, and after 339227baebb8STheodore Ts'o * that, we will force a minimum group preallocation size of 339327baebb8STheodore Ts'o * 32 clusters. This translates to 8 megs when the cluster 339427baebb8STheodore Ts'o * size is 256k, and 32 megs when the cluster size is 1 meg, 339527baebb8STheodore Ts'o * which seems reasonable as a default. 339627baebb8STheodore Ts'o */ 339727baebb8STheodore Ts'o sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> 339827baebb8STheodore Ts'o sbi->s_cluster_bits, 32); 3399d7a1fee1SDan Ehrenberg /* 3400d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 3401d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 3402d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 3403d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 3404d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 3405d7a1fee1SDan Ehrenberg * the stripes. 3406d7a1fee1SDan Ehrenberg */ 3407d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 3408d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 3409d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc, sbi->s_stripe); 3410d7a1fee1SDan Ehrenberg } 3411c9de560dSAlex Tomas 3412730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 3413c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 3414fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 3415029b10c5SAndrey Tsyvarev goto out; 3416c9de560dSAlex Tomas } 3417730c213cSEric Sandeen for_each_possible_cpu(i) { 3418c9de560dSAlex Tomas struct ext4_locality_group *lg; 3419730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 3420c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 34216be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 34226be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 3423c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 3424c9de560dSAlex Tomas } 3425c9de560dSAlex Tomas 3426196e402aSHarshad Shirwadkar if (blk_queue_nonrot(bdev_get_queue(sb->s_bdev))) 3427196e402aSHarshad Shirwadkar sbi->s_mb_max_linear_groups = 0; 3428196e402aSHarshad Shirwadkar else 3429196e402aSHarshad Shirwadkar sbi->s_mb_max_linear_groups = MB_DEFAULT_LINEAR_LIMIT; 343079a77c5aSYu Jian /* init file for buddy data */ 343179a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 34327aa0baeaSTao Ma if (ret != 0) 34337aa0baeaSTao Ma goto out_free_locality_groups; 343479a77c5aSYu Jian 34357aa0baeaSTao Ma return 0; 34367aa0baeaSTao Ma 34377aa0baeaSTao Ma out_free_locality_groups: 34387aa0baeaSTao Ma free_percpu(sbi->s_locality_groups); 34397aa0baeaSTao Ma sbi->s_locality_groups = NULL; 3440fb1813f4SCurt Wohlgemuth out: 3441196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders); 3442196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders_locks); 3443fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 34447aa0baeaSTao Ma sbi->s_mb_offsets = NULL; 3445fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 34467aa0baeaSTao Ma sbi->s_mb_maxs = NULL; 3447fb1813f4SCurt Wohlgemuth return ret; 3448c9de560dSAlex Tomas } 3449c9de560dSAlex Tomas 3450955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 3451d3df1453SRitesh Harjani static int ext4_mb_cleanup_pa(struct ext4_group_info *grp) 3452c9de560dSAlex Tomas { 3453c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3454c9de560dSAlex Tomas struct list_head *cur, *tmp; 3455c9de560dSAlex Tomas int count = 0; 3456c9de560dSAlex Tomas 3457c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 3458c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3459c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3460c9de560dSAlex Tomas count++; 3461688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 3462c9de560dSAlex Tomas } 3463d3df1453SRitesh Harjani return count; 3464c9de560dSAlex Tomas } 3465c9de560dSAlex Tomas 3466c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 3467c9de560dSAlex Tomas { 34688df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 3469c9de560dSAlex Tomas ext4_group_t i; 3470c9de560dSAlex Tomas int num_meta_group_infos; 3471df3da4eaSSuraj Jitindar Singh struct ext4_group_info *grinfo, ***group_info; 3472c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3473fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 3474d3df1453SRitesh Harjani int count; 3475c9de560dSAlex Tomas 3476c9de560dSAlex Tomas if (sbi->s_group_info) { 34778df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 34784b99faa2SKhazhismel Kumykov cond_resched(); 3479c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 3480a3450215SRitesh Harjani mb_group_bb_bitmap_free(grinfo); 3481c9de560dSAlex Tomas ext4_lock_group(sb, i); 3482d3df1453SRitesh Harjani count = ext4_mb_cleanup_pa(grinfo); 3483d3df1453SRitesh Harjani if (count) 3484d3df1453SRitesh Harjani mb_debug(sb, "mballoc: %d PAs left\n", 3485d3df1453SRitesh Harjani count); 3486c9de560dSAlex Tomas ext4_unlock_group(sb, i); 3487fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 3488c9de560dSAlex Tomas } 34898df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 3490c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 3491c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 3492df3da4eaSSuraj Jitindar Singh rcu_read_lock(); 3493df3da4eaSSuraj Jitindar Singh group_info = rcu_dereference(sbi->s_group_info); 3494c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 3495df3da4eaSSuraj Jitindar Singh kfree(group_info[i]); 3496df3da4eaSSuraj Jitindar Singh kvfree(group_info); 3497df3da4eaSSuraj Jitindar Singh rcu_read_unlock(); 3498c9de560dSAlex Tomas } 3499196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders); 3500196e402aSHarshad Shirwadkar kfree(sbi->s_mb_largest_free_orders_locks); 3501c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 3502c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 3503c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 3504c9de560dSAlex Tomas if (sbi->s_mb_stats) { 35059d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 35069d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 3507c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 3508c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 3509c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 35109d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 3511a6c75eafSHarshad Shirwadkar "mballoc: %u extents scanned, %u groups scanned, %u goal hits, " 35129d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 3513c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 3514a6c75eafSHarshad Shirwadkar atomic_read(&sbi->s_bal_groups_scanned), 3515c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 3516c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 3517c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 3518c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 35199d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 352067d25186SHarshad Shirwadkar "mballoc: %u generated and it took %llu", 352167d25186SHarshad Shirwadkar atomic_read(&sbi->s_mb_buddies_generated), 352267d25186SHarshad Shirwadkar atomic64_read(&sbi->s_mb_generation_time)); 35239d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 35249d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 3525c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 3526c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 3527c9de560dSAlex Tomas } 3528c9de560dSAlex Tomas 3529730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 3530c9de560dSAlex Tomas 3531c9de560dSAlex Tomas return 0; 3532c9de560dSAlex Tomas } 3533c9de560dSAlex Tomas 353477ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 3535a0154344SDaeho Jeong ext4_group_t block_group, ext4_grpblk_t cluster, int count, 3536a0154344SDaeho Jeong struct bio **biop) 35375c521830SJiaying Zhang { 35385c521830SJiaying Zhang ext4_fsblk_t discard_block; 35395c521830SJiaying Zhang 354084130193STheodore Ts'o discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + 354184130193STheodore Ts'o ext4_group_first_block_no(sb, block_group)); 354284130193STheodore Ts'o count = EXT4_C2B(EXT4_SB(sb), count); 35435c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 35445c521830SJiaying Zhang (unsigned long long) discard_block, count); 3545a0154344SDaeho Jeong if (biop) { 3546a0154344SDaeho Jeong return __blkdev_issue_discard(sb->s_bdev, 3547a0154344SDaeho Jeong (sector_t)discard_block << (sb->s_blocksize_bits - 9), 3548a0154344SDaeho Jeong (sector_t)count << (sb->s_blocksize_bits - 9), 3549a0154344SDaeho Jeong GFP_NOFS, 0, biop); 3550a0154344SDaeho Jeong } else 355193259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 35525c521830SJiaying Zhang } 35535c521830SJiaying Zhang 3554a0154344SDaeho Jeong static void ext4_free_data_in_buddy(struct super_block *sb, 3555a0154344SDaeho Jeong struct ext4_free_data *entry) 3556c9de560dSAlex Tomas { 3557c9de560dSAlex Tomas struct ext4_buddy e4b; 3558c894058dSAneesh Kumar K.V struct ext4_group_info *db; 3559d9f34504STheodore Ts'o int err, count = 0, count2 = 0; 3560c9de560dSAlex Tomas 3561d3df1453SRitesh Harjani mb_debug(sb, "gonna free %u blocks in group %u (0x%p):", 356218aadd47SBobi Jam entry->efd_count, entry->efd_group, entry); 3563c9de560dSAlex Tomas 356418aadd47SBobi Jam err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); 3565c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 3566c9de560dSAlex Tomas BUG_ON(err != 0); 3567c9de560dSAlex Tomas 3568d08854f5STheodore Ts'o spin_lock(&EXT4_SB(sb)->s_md_lock); 3569d08854f5STheodore Ts'o EXT4_SB(sb)->s_mb_free_pending -= entry->efd_count; 3570d08854f5STheodore Ts'o spin_unlock(&EXT4_SB(sb)->s_md_lock); 357118aadd47SBobi Jam 3572c894058dSAneesh Kumar K.V db = e4b.bd_info; 3573c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 357418aadd47SBobi Jam count += entry->efd_count; 3575c9de560dSAlex Tomas count2++; 357618aadd47SBobi Jam ext4_lock_group(sb, entry->efd_group); 3577c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 357818aadd47SBobi Jam rb_erase(&entry->efd_node, &(db->bb_free_root)); 357918aadd47SBobi Jam mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); 3580c9de560dSAlex Tomas 35813d56b8d2STao Ma /* 35823d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 35833d56b8d2STao Ma * ext4_trim_fs can trim it. 35843d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 35853d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 35863d56b8d2STao Ma */ 35873d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 35883d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 35893d56b8d2STao Ma 3590c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 3591c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 3592c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 3593c894058dSAneesh Kumar K.V */ 359409cbfeafSKirill A. Shutemov put_page(e4b.bd_buddy_page); 359509cbfeafSKirill A. Shutemov put_page(e4b.bd_bitmap_page); 3596c894058dSAneesh Kumar K.V } 359718aadd47SBobi Jam ext4_unlock_group(sb, entry->efd_group); 359818aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 3599e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3600c9de560dSAlex Tomas 3601d3df1453SRitesh Harjani mb_debug(sb, "freed %d blocks in %d structures\n", count, 3602d3df1453SRitesh Harjani count2); 3603c9de560dSAlex Tomas } 3604c9de560dSAlex Tomas 3605a0154344SDaeho Jeong /* 3606a0154344SDaeho Jeong * This function is called by the jbd2 layer once the commit has finished, 3607a0154344SDaeho Jeong * so we know we can free the blocks that were released with that commit. 3608a0154344SDaeho Jeong */ 3609a0154344SDaeho Jeong void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid) 3610a0154344SDaeho Jeong { 3611a0154344SDaeho Jeong struct ext4_sb_info *sbi = EXT4_SB(sb); 3612a0154344SDaeho Jeong struct ext4_free_data *entry, *tmp; 3613a0154344SDaeho Jeong struct bio *discard_bio = NULL; 3614a0154344SDaeho Jeong struct list_head freed_data_list; 3615a0154344SDaeho Jeong struct list_head *cut_pos = NULL; 3616a0154344SDaeho Jeong int err; 3617a0154344SDaeho Jeong 3618a0154344SDaeho Jeong INIT_LIST_HEAD(&freed_data_list); 3619a0154344SDaeho Jeong 3620a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 3621a0154344SDaeho Jeong list_for_each_entry(entry, &sbi->s_freed_data_list, efd_list) { 3622a0154344SDaeho Jeong if (entry->efd_tid != commit_tid) 3623a0154344SDaeho Jeong break; 3624a0154344SDaeho Jeong cut_pos = &entry->efd_list; 3625a0154344SDaeho Jeong } 3626a0154344SDaeho Jeong if (cut_pos) 3627a0154344SDaeho Jeong list_cut_position(&freed_data_list, &sbi->s_freed_data_list, 3628a0154344SDaeho Jeong cut_pos); 3629a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 3630a0154344SDaeho Jeong 3631a0154344SDaeho Jeong if (test_opt(sb, DISCARD)) { 3632a0154344SDaeho Jeong list_for_each_entry(entry, &freed_data_list, efd_list) { 3633a0154344SDaeho Jeong err = ext4_issue_discard(sb, entry->efd_group, 3634a0154344SDaeho Jeong entry->efd_start_cluster, 3635a0154344SDaeho Jeong entry->efd_count, 3636a0154344SDaeho Jeong &discard_bio); 3637a0154344SDaeho Jeong if (err && err != -EOPNOTSUPP) { 3638a0154344SDaeho Jeong ext4_msg(sb, KERN_WARNING, "discard request in" 3639a0154344SDaeho Jeong " group:%d block:%d count:%d failed" 3640a0154344SDaeho Jeong " with %d", entry->efd_group, 3641a0154344SDaeho Jeong entry->efd_start_cluster, 3642a0154344SDaeho Jeong entry->efd_count, err); 3643a0154344SDaeho Jeong } else if (err == -EOPNOTSUPP) 3644a0154344SDaeho Jeong break; 3645a0154344SDaeho Jeong } 3646a0154344SDaeho Jeong 3647e4510577SDaeho Jeong if (discard_bio) { 3648a0154344SDaeho Jeong submit_bio_wait(discard_bio); 3649e4510577SDaeho Jeong bio_put(discard_bio); 3650e4510577SDaeho Jeong } 3651a0154344SDaeho Jeong } 3652a0154344SDaeho Jeong 3653a0154344SDaeho Jeong list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list) 3654a0154344SDaeho Jeong ext4_free_data_in_buddy(sb, entry); 3655a0154344SDaeho Jeong } 3656a0154344SDaeho Jeong 36575dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 3658c9de560dSAlex Tomas { 365916828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 366016828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3661c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 3662f283529aSRitesh Harjani goto out; 3663c9de560dSAlex Tomas 366416828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 366516828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3666f283529aSRitesh Harjani if (ext4_ac_cachep == NULL) 3667f283529aSRitesh Harjani goto out_pa_free; 3668c894058dSAneesh Kumar K.V 366918aadd47SBobi Jam ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, 367016828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 3671f283529aSRitesh Harjani if (ext4_free_data_cachep == NULL) 3672f283529aSRitesh Harjani goto out_ac_free; 3673f283529aSRitesh Harjani 3674c9de560dSAlex Tomas return 0; 3675f283529aSRitesh Harjani 3676f283529aSRitesh Harjani out_ac_free: 3677f283529aSRitesh Harjani kmem_cache_destroy(ext4_ac_cachep); 3678f283529aSRitesh Harjani out_pa_free: 3679f283529aSRitesh Harjani kmem_cache_destroy(ext4_pspace_cachep); 3680f283529aSRitesh Harjani out: 3681f283529aSRitesh Harjani return -ENOMEM; 3682c9de560dSAlex Tomas } 3683c9de560dSAlex Tomas 36845dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 3685c9de560dSAlex Tomas { 36863e03f9caSJesper Dangaard Brouer /* 36873e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 36883e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 36893e03f9caSJesper Dangaard Brouer */ 36903e03f9caSJesper Dangaard Brouer rcu_barrier(); 3691c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 3692256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 369318aadd47SBobi Jam kmem_cache_destroy(ext4_free_data_cachep); 36942892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 3695c9de560dSAlex Tomas } 3696c9de560dSAlex Tomas 3697c9de560dSAlex Tomas 3698c9de560dSAlex Tomas /* 369973b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 3700c9de560dSAlex Tomas * Returns 0 if success or error code 3701c9de560dSAlex Tomas */ 37024ddfef7bSEric Sandeen static noinline_for_stack int 37034ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 370453accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 3705c9de560dSAlex Tomas { 3706c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3707c9de560dSAlex Tomas struct ext4_group_desc *gdp; 3708c9de560dSAlex Tomas struct buffer_head *gdp_bh; 3709c9de560dSAlex Tomas struct ext4_sb_info *sbi; 3710c9de560dSAlex Tomas struct super_block *sb; 3711c9de560dSAlex Tomas ext4_fsblk_t block; 3712519deca0SAneesh Kumar K.V int err, len; 3713c9de560dSAlex Tomas 3714c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3715c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 3716c9de560dSAlex Tomas 3717c9de560dSAlex Tomas sb = ac->ac_sb; 3718c9de560dSAlex Tomas sbi = EXT4_SB(sb); 3719c9de560dSAlex Tomas 3720574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 37219008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 37229008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 37239008a58eSDarrick J. Wong bitmap_bh = NULL; 3724c9de560dSAlex Tomas goto out_err; 37259008a58eSDarrick J. Wong } 3726c9de560dSAlex Tomas 37275d601255Sliang xie BUFFER_TRACE(bitmap_bh, "getting write access"); 3728c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 3729c9de560dSAlex Tomas if (err) 3730c9de560dSAlex Tomas goto out_err; 3731c9de560dSAlex Tomas 3732c9de560dSAlex Tomas err = -EIO; 3733c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 3734c9de560dSAlex Tomas if (!gdp) 3735c9de560dSAlex Tomas goto out_err; 3736c9de560dSAlex Tomas 3737a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 3738021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, gdp)); 373903cddb80SAneesh Kumar K.V 37405d601255Sliang xie BUFFER_TRACE(gdp_bh, "get_write_access"); 3741c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 3742c9de560dSAlex Tomas if (err) 3743c9de560dSAlex Tomas goto out_err; 3744c9de560dSAlex Tomas 3745bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3746c9de560dSAlex Tomas 374753accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3748ce9f24ccSJan Kara if (!ext4_inode_block_valid(ac->ac_inode, block, len)) { 374912062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 37501084f252STheodore Ts'o "fs metadata", block, block+len); 3751519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 3752554a5cccSVegard Nossum * Fix the bitmap and return EFSCORRUPTED 3753519deca0SAneesh Kumar K.V * We leak some of the blocks here. 3754519deca0SAneesh Kumar K.V */ 3755955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3756c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3757519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 3758955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 37590390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3760519deca0SAneesh Kumar K.V if (!err) 3761554a5cccSVegard Nossum err = -EFSCORRUPTED; 3762519deca0SAneesh Kumar K.V goto out_err; 3763c9de560dSAlex Tomas } 3764955ce5f5SAneesh Kumar K.V 3765955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3766c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 3767c9de560dSAlex Tomas { 3768c9de560dSAlex Tomas int i; 3769c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 3770c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 3771c9de560dSAlex Tomas bitmap_bh->b_data)); 3772c9de560dSAlex Tomas } 3773c9de560dSAlex Tomas } 3774c9de560dSAlex Tomas #endif 3775c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 3776c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 37778844618dSTheodore Ts'o if (ext4_has_group_desc_csum(sb) && 37788844618dSTheodore Ts'o (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 3779c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 3780021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, 3781cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, 3782560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 3783c9de560dSAlex Tomas } 3784021b65bbSTheodore Ts'o len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; 3785021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, len); 378679f1ba49STao Ma ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); 3787feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); 3788955ce5f5SAneesh Kumar K.V 3789955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 379057042651STheodore Ts'o percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); 3791d2a17637SMingming Cao /* 37926bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 3793d2a17637SMingming Cao */ 37946bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 37956bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 379657042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 379757042651STheodore Ts'o reserv_clstrs); 3798c9de560dSAlex Tomas 3799772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 3800772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 3801772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 380290ba983fSTheodore Ts'o atomic64_sub(ac->ac_b_ex.fe_len, 38037c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 38047c990728SSuraj Jitindar Singh flex_group)->free_clusters); 3805772cb7c8SJose R. Santos } 3806772cb7c8SJose R. Santos 38070390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3808c9de560dSAlex Tomas if (err) 3809c9de560dSAlex Tomas goto out_err; 38100390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 3811c9de560dSAlex Tomas 3812c9de560dSAlex Tomas out_err: 381342a10addSAneesh Kumar K.V brelse(bitmap_bh); 3814c9de560dSAlex Tomas return err; 3815c9de560dSAlex Tomas } 3816c9de560dSAlex Tomas 3817c9de560dSAlex Tomas /* 38188016e29fSHarshad Shirwadkar * Idempotent helper for Ext4 fast commit replay path to set the state of 38198016e29fSHarshad Shirwadkar * blocks in bitmaps and update counters. 38208016e29fSHarshad Shirwadkar */ 38218016e29fSHarshad Shirwadkar void ext4_mb_mark_bb(struct super_block *sb, ext4_fsblk_t block, 38228016e29fSHarshad Shirwadkar int len, int state) 38238016e29fSHarshad Shirwadkar { 38248016e29fSHarshad Shirwadkar struct buffer_head *bitmap_bh = NULL; 38258016e29fSHarshad Shirwadkar struct ext4_group_desc *gdp; 38268016e29fSHarshad Shirwadkar struct buffer_head *gdp_bh; 38278016e29fSHarshad Shirwadkar struct ext4_sb_info *sbi = EXT4_SB(sb); 38288016e29fSHarshad Shirwadkar ext4_group_t group; 38298016e29fSHarshad Shirwadkar ext4_grpblk_t blkoff; 38308016e29fSHarshad Shirwadkar int i, clen, err; 38318016e29fSHarshad Shirwadkar int already; 38328016e29fSHarshad Shirwadkar 38338016e29fSHarshad Shirwadkar clen = EXT4_B2C(sbi, len); 38348016e29fSHarshad Shirwadkar 38358016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, block, &group, &blkoff); 38368016e29fSHarshad Shirwadkar bitmap_bh = ext4_read_block_bitmap(sb, group); 38378016e29fSHarshad Shirwadkar if (IS_ERR(bitmap_bh)) { 38388016e29fSHarshad Shirwadkar err = PTR_ERR(bitmap_bh); 38398016e29fSHarshad Shirwadkar bitmap_bh = NULL; 38408016e29fSHarshad Shirwadkar goto out_err; 38418016e29fSHarshad Shirwadkar } 38428016e29fSHarshad Shirwadkar 38438016e29fSHarshad Shirwadkar err = -EIO; 38448016e29fSHarshad Shirwadkar gdp = ext4_get_group_desc(sb, group, &gdp_bh); 38458016e29fSHarshad Shirwadkar if (!gdp) 38468016e29fSHarshad Shirwadkar goto out_err; 38478016e29fSHarshad Shirwadkar 38488016e29fSHarshad Shirwadkar ext4_lock_group(sb, group); 38498016e29fSHarshad Shirwadkar already = 0; 38508016e29fSHarshad Shirwadkar for (i = 0; i < clen; i++) 38518016e29fSHarshad Shirwadkar if (!mb_test_bit(blkoff + i, bitmap_bh->b_data) == !state) 38528016e29fSHarshad Shirwadkar already++; 38538016e29fSHarshad Shirwadkar 38548016e29fSHarshad Shirwadkar if (state) 38558016e29fSHarshad Shirwadkar ext4_set_bits(bitmap_bh->b_data, blkoff, clen); 38568016e29fSHarshad Shirwadkar else 38578016e29fSHarshad Shirwadkar mb_test_and_clear_bits(bitmap_bh->b_data, blkoff, clen); 38588016e29fSHarshad Shirwadkar if (ext4_has_group_desc_csum(sb) && 38598016e29fSHarshad Shirwadkar (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT))) { 38608016e29fSHarshad Shirwadkar gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 38618016e29fSHarshad Shirwadkar ext4_free_group_clusters_set(sb, gdp, 38628016e29fSHarshad Shirwadkar ext4_free_clusters_after_init(sb, 38638016e29fSHarshad Shirwadkar group, gdp)); 38648016e29fSHarshad Shirwadkar } 38658016e29fSHarshad Shirwadkar if (state) 38668016e29fSHarshad Shirwadkar clen = ext4_free_group_clusters(sb, gdp) - clen + already; 38678016e29fSHarshad Shirwadkar else 38688016e29fSHarshad Shirwadkar clen = ext4_free_group_clusters(sb, gdp) + clen - already; 38698016e29fSHarshad Shirwadkar 38708016e29fSHarshad Shirwadkar ext4_free_group_clusters_set(sb, gdp, clen); 38718016e29fSHarshad Shirwadkar ext4_block_bitmap_csum_set(sb, group, gdp, bitmap_bh); 38728016e29fSHarshad Shirwadkar ext4_group_desc_csum_set(sb, group, gdp); 38738016e29fSHarshad Shirwadkar 38748016e29fSHarshad Shirwadkar ext4_unlock_group(sb, group); 38758016e29fSHarshad Shirwadkar 38768016e29fSHarshad Shirwadkar if (sbi->s_log_groups_per_flex) { 38778016e29fSHarshad Shirwadkar ext4_group_t flex_group = ext4_flex_group(sbi, group); 38788016e29fSHarshad Shirwadkar 38798016e29fSHarshad Shirwadkar atomic64_sub(len, 38808016e29fSHarshad Shirwadkar &sbi_array_rcu_deref(sbi, s_flex_groups, 38818016e29fSHarshad Shirwadkar flex_group)->free_clusters); 38828016e29fSHarshad Shirwadkar } 38838016e29fSHarshad Shirwadkar 38848016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh); 38858016e29fSHarshad Shirwadkar if (err) 38868016e29fSHarshad Shirwadkar goto out_err; 38878016e29fSHarshad Shirwadkar sync_dirty_buffer(bitmap_bh); 38888016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, gdp_bh); 38898016e29fSHarshad Shirwadkar sync_dirty_buffer(gdp_bh); 38908016e29fSHarshad Shirwadkar 38918016e29fSHarshad Shirwadkar out_err: 38928016e29fSHarshad Shirwadkar brelse(bitmap_bh); 38938016e29fSHarshad Shirwadkar } 38948016e29fSHarshad Shirwadkar 38958016e29fSHarshad Shirwadkar /* 3896c9de560dSAlex Tomas * here we normalize request for locality group 3897d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 3898d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 3899d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 3900b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 3901c9de560dSAlex Tomas * 3902c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 3903c9de560dSAlex Tomas */ 3904c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 3905c9de560dSAlex Tomas { 3906c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3907c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 3908c9de560dSAlex Tomas 3909c9de560dSAlex Tomas BUG_ON(lg == NULL); 3910c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 3911d3df1453SRitesh Harjani mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len); 3912c9de560dSAlex Tomas } 3913c9de560dSAlex Tomas 3914c9de560dSAlex Tomas /* 3915c9de560dSAlex Tomas * Normalization means making request better in terms of 3916c9de560dSAlex Tomas * size and alignment 3917c9de560dSAlex Tomas */ 39184ddfef7bSEric Sandeen static noinline_for_stack void 39194ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 3920c9de560dSAlex Tomas struct ext4_allocation_request *ar) 3921c9de560dSAlex Tomas { 392253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3923c9de560dSAlex Tomas int bsbits, max; 3924c9de560dSAlex Tomas ext4_lblk_t end; 39251592d2c5SCurt Wohlgemuth loff_t size, start_off; 39261592d2c5SCurt Wohlgemuth loff_t orig_size __maybe_unused; 39275a0790c2SAndi Kleen ext4_lblk_t start; 3928c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 39299a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 3930c9de560dSAlex Tomas 3931c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 3932c9de560dSAlex Tomas do not need preallocation */ 3933c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3934c9de560dSAlex Tomas return; 3935c9de560dSAlex Tomas 3936c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3937c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3938c9de560dSAlex Tomas return; 3939c9de560dSAlex Tomas 3940c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3941c9de560dSAlex Tomas * required (it's a tail, for example) */ 3942c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3943c9de560dSAlex Tomas return; 3944c9de560dSAlex Tomas 3945c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3946c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3947c9de560dSAlex Tomas return ; 3948c9de560dSAlex Tomas } 3949c9de560dSAlex Tomas 3950c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3951c9de560dSAlex Tomas 3952c9de560dSAlex Tomas /* first, let's learn actual file size 3953c9de560dSAlex Tomas * given current request is allocated */ 395453accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 3955c9de560dSAlex Tomas size = size << bsbits; 3956c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3957c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 39585a0790c2SAndi Kleen orig_size = size; 3959c9de560dSAlex Tomas 39601930479cSValerie Clement /* max size of free chunks */ 39611930479cSValerie Clement max = 2 << bsbits; 3962c9de560dSAlex Tomas 39631930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 39641930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 3965c9de560dSAlex Tomas 3966c9de560dSAlex Tomas /* first, try to predict filesize */ 3967c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3968c9de560dSAlex Tomas start_off = 0; 3969c9de560dSAlex Tomas if (size <= 16 * 1024) { 3970c9de560dSAlex Tomas size = 16 * 1024; 3971c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3972c9de560dSAlex Tomas size = 32 * 1024; 3973c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3974c9de560dSAlex Tomas size = 64 * 1024; 3975c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3976c9de560dSAlex Tomas size = 128 * 1024; 3977c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3978c9de560dSAlex Tomas size = 256 * 1024; 3979c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3980c9de560dSAlex Tomas size = 512 * 1024; 3981c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3982c9de560dSAlex Tomas size = 1024 * 1024; 39831930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 3984c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 39851930479cSValerie Clement (21 - bsbits)) << 21; 39861930479cSValerie Clement size = 2 * 1024 * 1024; 39871930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 3988c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3989c9de560dSAlex Tomas (22 - bsbits)) << 22; 3990c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3991c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 39921930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 3993c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3994c9de560dSAlex Tomas (23 - bsbits)) << 23; 3995c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3996c9de560dSAlex Tomas } else { 3997c9de560dSAlex Tomas start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits; 3998b27b1535SXiaoguang Wang size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb), 3999b27b1535SXiaoguang Wang ac->ac_o_ex.fe_len) << bsbits; 4000c9de560dSAlex Tomas } 40015a0790c2SAndi Kleen size = size >> bsbits; 40025a0790c2SAndi Kleen start = start_off >> bsbits; 4003c9de560dSAlex Tomas 4004c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 4005c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 4006c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 4007c9de560dSAlex Tomas start = ar->lleft + 1; 4008c9de560dSAlex Tomas } 4009c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 4010c9de560dSAlex Tomas size -= start + size - ar->lright; 4011c9de560dSAlex Tomas 4012cd648b8aSJan Kara /* 4013cd648b8aSJan Kara * Trim allocation request for filesystems with artificially small 4014cd648b8aSJan Kara * groups. 4015cd648b8aSJan Kara */ 4016cd648b8aSJan Kara if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) 4017cd648b8aSJan Kara size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb); 4018cd648b8aSJan Kara 4019c9de560dSAlex Tomas end = start + size; 4020c9de560dSAlex Tomas 4021c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 4022c9de560dSAlex Tomas rcu_read_lock(); 40239a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 4024498e5f24STheodore Ts'o ext4_lblk_t pa_end; 4025c9de560dSAlex Tomas 4026c9de560dSAlex Tomas if (pa->pa_deleted) 4027c9de560dSAlex Tomas continue; 4028c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4029c9de560dSAlex Tomas if (pa->pa_deleted) { 4030c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4031c9de560dSAlex Tomas continue; 4032c9de560dSAlex Tomas } 4033c9de560dSAlex Tomas 403453accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 403553accfa9STheodore Ts'o pa->pa_len); 4036c9de560dSAlex Tomas 4037c9de560dSAlex Tomas /* PA must not overlap original request */ 4038c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 4039c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 4040c9de560dSAlex Tomas 404138877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 404238877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 4043c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4044c9de560dSAlex Tomas continue; 4045c9de560dSAlex Tomas } 4046c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 4047c9de560dSAlex Tomas 404838877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 4049c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 4050c9de560dSAlex Tomas BUG_ON(pa_end < start); 4051c9de560dSAlex Tomas start = pa_end; 405238877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 4053c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 4054c9de560dSAlex Tomas end = pa->pa_lstart; 4055c9de560dSAlex Tomas } 4056c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4057c9de560dSAlex Tomas } 4058c9de560dSAlex Tomas rcu_read_unlock(); 4059c9de560dSAlex Tomas size = end - start; 4060c9de560dSAlex Tomas 4061c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 4062c9de560dSAlex Tomas rcu_read_lock(); 40639a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 4064498e5f24STheodore Ts'o ext4_lblk_t pa_end; 406553accfa9STheodore Ts'o 4066c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4067c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 406853accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 406953accfa9STheodore Ts'o pa->pa_len); 4070c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 4071c9de560dSAlex Tomas } 4072c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4073c9de560dSAlex Tomas } 4074c9de560dSAlex Tomas rcu_read_unlock(); 4075c9de560dSAlex Tomas 4076c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 4077c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 40789d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 40799d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 4080c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 4081c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 4082dfe076c1SDmitry Monakhov BUG(); 4083c9de560dSAlex Tomas } 4084b5b60778SMaurizio Lombardi BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 4085c9de560dSAlex Tomas 4086c9de560dSAlex Tomas /* now prepare goal request */ 4087c9de560dSAlex Tomas 4088c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 4089c9de560dSAlex Tomas * placement or satisfy big request as is */ 4090c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 409153accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 4092c9de560dSAlex Tomas 4093c9de560dSAlex Tomas /* define goal start in order to merge */ 4094c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 4095c9de560dSAlex Tomas /* merge to the right */ 4096c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 4097c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 4098c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 4099c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 4100c9de560dSAlex Tomas } 4101c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 4102c9de560dSAlex Tomas /* merge to the left */ 4103c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 4104c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 4105c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 4106c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 4107c9de560dSAlex Tomas } 4108c9de560dSAlex Tomas 4109d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size, 4110d3df1453SRitesh Harjani orig_size, start); 4111c9de560dSAlex Tomas } 4112c9de560dSAlex Tomas 4113c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 4114c9de560dSAlex Tomas { 4115c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4116c9de560dSAlex Tomas 4117a6c75eafSHarshad Shirwadkar if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) { 4118c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 4119c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 4120291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 4121c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 4122c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 4123a6c75eafSHarshad Shirwadkar atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned); 4124c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 4125c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 4126c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 4127c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 4128c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 4129c9de560dSAlex Tomas } 4130c9de560dSAlex Tomas 4131296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 4132296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 4133296c355cSTheodore Ts'o else 4134296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 4135c9de560dSAlex Tomas } 4136c9de560dSAlex Tomas 4137c9de560dSAlex Tomas /* 4138b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 4139b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 4140b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 4141b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 4142b844167eSCurt Wohlgemuth */ 4143b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 4144b844167eSCurt Wohlgemuth { 4145b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 414686f0afd4STheodore Ts'o struct ext4_buddy e4b; 414786f0afd4STheodore Ts'o int err; 4148b844167eSCurt Wohlgemuth 414986f0afd4STheodore Ts'o if (pa == NULL) { 4150c99d1e6eSTheodore Ts'o if (ac->ac_f_ex.fe_len == 0) 4151c99d1e6eSTheodore Ts'o return; 415286f0afd4STheodore Ts'o err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b); 415386f0afd4STheodore Ts'o if (err) { 415486f0afd4STheodore Ts'o /* 415586f0afd4STheodore Ts'o * This should never happen since we pin the 415686f0afd4STheodore Ts'o * pages in the ext4_allocation_context so 415786f0afd4STheodore Ts'o * ext4_mb_load_buddy() should never fail. 415886f0afd4STheodore Ts'o */ 415986f0afd4STheodore Ts'o WARN(1, "mb_load_buddy failed (%d)", err); 416086f0afd4STheodore Ts'o return; 416186f0afd4STheodore Ts'o } 416286f0afd4STheodore Ts'o ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 416386f0afd4STheodore Ts'o mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start, 416486f0afd4STheodore Ts'o ac->ac_f_ex.fe_len); 416586f0afd4STheodore Ts'o ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group); 4166c99d1e6eSTheodore Ts'o ext4_mb_unload_buddy(&e4b); 416786f0afd4STheodore Ts'o return; 416886f0afd4STheodore Ts'o } 416986f0afd4STheodore Ts'o if (pa->pa_type == MB_INODE_PA) 4170400db9d3SZheng Liu pa->pa_free += ac->ac_b_ex.fe_len; 4171b844167eSCurt Wohlgemuth } 4172b844167eSCurt Wohlgemuth 4173b844167eSCurt Wohlgemuth /* 4174c9de560dSAlex Tomas * use blocks preallocated to inode 4175c9de560dSAlex Tomas */ 4176c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 4177c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 4178c9de560dSAlex Tomas { 417953accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4180c9de560dSAlex Tomas ext4_fsblk_t start; 4181c9de560dSAlex Tomas ext4_fsblk_t end; 4182c9de560dSAlex Tomas int len; 4183c9de560dSAlex Tomas 4184c9de560dSAlex Tomas /* found preallocated blocks, use them */ 4185c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 418653accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 418753accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 418853accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 4189c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 4190c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 4191c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 4192c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 4193c9de560dSAlex Tomas ac->ac_pa = pa; 4194c9de560dSAlex Tomas 4195c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 419653accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 4197c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 4198c9de560dSAlex Tomas pa->pa_free -= len; 4199c9de560dSAlex Tomas 4200d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa); 4201c9de560dSAlex Tomas } 4202c9de560dSAlex Tomas 4203c9de560dSAlex Tomas /* 4204c9de560dSAlex Tomas * use blocks preallocated to locality group 4205c9de560dSAlex Tomas */ 4206c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 4207c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 4208c9de560dSAlex Tomas { 420903cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 42106be2ded1SAneesh Kumar K.V 4211c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 4212c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 4213c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 4214c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 4215c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 4216c9de560dSAlex Tomas ac->ac_pa = pa; 4217c9de560dSAlex Tomas 4218c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 421926346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 4220c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 422126346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 422226346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 4223c9de560dSAlex Tomas */ 4224d3df1453SRitesh Harjani mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n", 4225d3df1453SRitesh Harjani pa->pa_lstart-len, len, pa); 4226c9de560dSAlex Tomas } 4227c9de560dSAlex Tomas 4228c9de560dSAlex Tomas /* 42295e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 42305e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 42315e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 42325e745b04SAneesh Kumar K.V * from the goal block. 42335e745b04SAneesh Kumar K.V */ 42345e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 42355e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 42365e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 42375e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 42385e745b04SAneesh Kumar K.V { 42395e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 42405e745b04SAneesh Kumar K.V 42415e745b04SAneesh Kumar K.V if (cpa == NULL) { 42425e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 42435e745b04SAneesh Kumar K.V return pa; 42445e745b04SAneesh Kumar K.V } 424579211c8eSAndrew Morton cur_distance = abs(goal_block - cpa->pa_pstart); 424679211c8eSAndrew Morton new_distance = abs(goal_block - pa->pa_pstart); 42475e745b04SAneesh Kumar K.V 42485a54b2f1SColy Li if (cur_distance <= new_distance) 42495e745b04SAneesh Kumar K.V return cpa; 42505e745b04SAneesh Kumar K.V 42515e745b04SAneesh Kumar K.V /* drop the previous reference */ 42525e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 42535e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 42545e745b04SAneesh Kumar K.V return pa; 42555e745b04SAneesh Kumar K.V } 42565e745b04SAneesh Kumar K.V 42575e745b04SAneesh Kumar K.V /* 4258c9de560dSAlex Tomas * search goal blocks in preallocated space 4259c9de560dSAlex Tomas */ 42604fca8f07SRitesh Harjani static noinline_for_stack bool 42614ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 4262c9de560dSAlex Tomas { 426353accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 42646be2ded1SAneesh Kumar K.V int order, i; 4265c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 4266c9de560dSAlex Tomas struct ext4_locality_group *lg; 42675e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 42685e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 4269c9de560dSAlex Tomas 4270c9de560dSAlex Tomas /* only data can be preallocated */ 4271c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 42724fca8f07SRitesh Harjani return false; 4273c9de560dSAlex Tomas 4274c9de560dSAlex Tomas /* first, try per-file preallocation */ 4275c9de560dSAlex Tomas rcu_read_lock(); 42769a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 4277c9de560dSAlex Tomas 4278c9de560dSAlex Tomas /* all fields in this condition don't change, 4279c9de560dSAlex Tomas * so we can skip locking for them */ 4280c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 428153accfa9STheodore Ts'o ac->ac_o_ex.fe_logical >= (pa->pa_lstart + 428253accfa9STheodore Ts'o EXT4_C2B(sbi, pa->pa_len))) 4283c9de560dSAlex Tomas continue; 4284c9de560dSAlex Tomas 4285fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 428612e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 428753accfa9STheodore Ts'o (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > 428853accfa9STheodore Ts'o EXT4_MAX_BLOCK_FILE_PHYS)) 4289fb0a387dSEric Sandeen continue; 4290fb0a387dSEric Sandeen 4291c9de560dSAlex Tomas /* found preallocated blocks, use them */ 4292c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4293c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 4294c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 4295c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 4296c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4297c9de560dSAlex Tomas ac->ac_criteria = 10; 4298c9de560dSAlex Tomas rcu_read_unlock(); 42994fca8f07SRitesh Harjani return true; 4300c9de560dSAlex Tomas } 4301c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4302c9de560dSAlex Tomas } 4303c9de560dSAlex Tomas rcu_read_unlock(); 4304c9de560dSAlex Tomas 4305c9de560dSAlex Tomas /* can we use group allocation? */ 4306c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 43074fca8f07SRitesh Harjani return false; 4308c9de560dSAlex Tomas 4309c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 4310c9de560dSAlex Tomas lg = ac->ac_lg; 4311c9de560dSAlex Tomas if (lg == NULL) 43124fca8f07SRitesh Harjani return false; 43136be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 43146be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 43156be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 43166be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 4317c9de560dSAlex Tomas 4318bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 43195e745b04SAneesh Kumar K.V /* 43205e745b04SAneesh Kumar K.V * search for the prealloc space that is having 43215e745b04SAneesh Kumar K.V * minimal distance from the goal block. 43225e745b04SAneesh Kumar K.V */ 43236be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 4324c9de560dSAlex Tomas rcu_read_lock(); 43256be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 43266be2ded1SAneesh Kumar K.V pa_inode_list) { 4327c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 43286be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 43296be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 43305e745b04SAneesh Kumar K.V 43315e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 43325e745b04SAneesh Kumar K.V pa, cpa); 43335e745b04SAneesh Kumar K.V } 4334c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 43355e745b04SAneesh Kumar K.V } 43365e745b04SAneesh Kumar K.V rcu_read_unlock(); 43375e745b04SAneesh Kumar K.V } 43385e745b04SAneesh Kumar K.V if (cpa) { 43395e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 4340c9de560dSAlex Tomas ac->ac_criteria = 20; 43414fca8f07SRitesh Harjani return true; 4342c9de560dSAlex Tomas } 43434fca8f07SRitesh Harjani return false; 4344c9de560dSAlex Tomas } 4345c9de560dSAlex Tomas 4346c9de560dSAlex Tomas /* 43477a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 43487a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 43497a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 4350955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 43517a2fcbf7SAneesh Kumar K.V */ 43527a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 43537a2fcbf7SAneesh Kumar K.V ext4_group_t group) 43547a2fcbf7SAneesh Kumar K.V { 43557a2fcbf7SAneesh Kumar K.V struct rb_node *n; 43567a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 43577a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 43587a2fcbf7SAneesh Kumar K.V 43597a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 43607a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 43617a2fcbf7SAneesh Kumar K.V 43627a2fcbf7SAneesh Kumar K.V while (n) { 436318aadd47SBobi Jam entry = rb_entry(n, struct ext4_free_data, efd_node); 436418aadd47SBobi Jam ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); 43657a2fcbf7SAneesh Kumar K.V n = rb_next(n); 43667a2fcbf7SAneesh Kumar K.V } 43677a2fcbf7SAneesh Kumar K.V return; 43687a2fcbf7SAneesh Kumar K.V } 43697a2fcbf7SAneesh Kumar K.V 43707a2fcbf7SAneesh Kumar K.V /* 4371c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 4372c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 4373955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 4374c9de560dSAlex Tomas */ 4375089ceeccSEric Sandeen static noinline_for_stack 4376089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 4377c9de560dSAlex Tomas ext4_group_t group) 4378c9de560dSAlex Tomas { 4379c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 4380c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4381c9de560dSAlex Tomas struct list_head *cur; 4382c9de560dSAlex Tomas ext4_group_t groupnr; 4383c9de560dSAlex Tomas ext4_grpblk_t start; 4384c9de560dSAlex Tomas int preallocated = 0; 4385c9de560dSAlex Tomas int len; 4386c9de560dSAlex Tomas 4387c9de560dSAlex Tomas /* all form of preallocation discards first load group, 4388c9de560dSAlex Tomas * so the only competing code is preallocation use. 4389c9de560dSAlex Tomas * we don't need any locking here 4390c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 4391c9de560dSAlex Tomas * otherwise we could leave used blocks available for 4392c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 4393c9de560dSAlex Tomas * is dropping preallocation 4394c9de560dSAlex Tomas */ 4395c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4396c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 4397c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4398c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4399c9de560dSAlex Tomas &groupnr, &start); 4400c9de560dSAlex Tomas len = pa->pa_len; 4401c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4402c9de560dSAlex Tomas if (unlikely(len == 0)) 4403c9de560dSAlex Tomas continue; 4404c9de560dSAlex Tomas BUG_ON(groupnr != group); 4405c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, start, len); 4406c9de560dSAlex Tomas preallocated += len; 4407c9de560dSAlex Tomas } 4408d3df1453SRitesh Harjani mb_debug(sb, "preallocated %d for group %u\n", preallocated, group); 4409c9de560dSAlex Tomas } 4410c9de560dSAlex Tomas 441127bc446eSbrookxu static void ext4_mb_mark_pa_deleted(struct super_block *sb, 441227bc446eSbrookxu struct ext4_prealloc_space *pa) 441327bc446eSbrookxu { 441427bc446eSbrookxu struct ext4_inode_info *ei; 441527bc446eSbrookxu 441627bc446eSbrookxu if (pa->pa_deleted) { 441727bc446eSbrookxu ext4_warning(sb, "deleted pa, type:%d, pblk:%llu, lblk:%u, len:%d\n", 441827bc446eSbrookxu pa->pa_type, pa->pa_pstart, pa->pa_lstart, 441927bc446eSbrookxu pa->pa_len); 442027bc446eSbrookxu return; 442127bc446eSbrookxu } 442227bc446eSbrookxu 442327bc446eSbrookxu pa->pa_deleted = 1; 442427bc446eSbrookxu 442527bc446eSbrookxu if (pa->pa_type == MB_INODE_PA) { 442627bc446eSbrookxu ei = EXT4_I(pa->pa_inode); 442727bc446eSbrookxu atomic_dec(&ei->i_prealloc_active); 442827bc446eSbrookxu } 442927bc446eSbrookxu } 443027bc446eSbrookxu 4431c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 4432c9de560dSAlex Tomas { 4433c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4434c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 44354e8d2139SJunho Ryu 44364e8d2139SJunho Ryu BUG_ON(atomic_read(&pa->pa_count)); 44374e8d2139SJunho Ryu BUG_ON(pa->pa_deleted == 0); 4438c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 4439c9de560dSAlex Tomas } 4440c9de560dSAlex Tomas 4441c9de560dSAlex Tomas /* 4442c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 4443c9de560dSAlex Tomas * if this was the last reference and the space is consumed 4444c9de560dSAlex Tomas */ 4445c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 4446c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 4447c9de560dSAlex Tomas { 4448a9df9a49STheodore Ts'o ext4_group_t grp; 4449d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 4450c9de560dSAlex Tomas 4451c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 4452c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 44534e8d2139SJunho Ryu if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { 44544e8d2139SJunho Ryu spin_unlock(&pa->pa_lock); 44554e8d2139SJunho Ryu return; 44564e8d2139SJunho Ryu } 44574e8d2139SJunho Ryu 4458c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 4459c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4460c9de560dSAlex Tomas return; 4461c9de560dSAlex Tomas } 4462c9de560dSAlex Tomas 446327bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 4464c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4465c9de560dSAlex Tomas 4466d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 4467cc0fb9adSAneesh Kumar K.V /* 4468cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 4469cc0fb9adSAneesh Kumar K.V * next group when pa is used up 4470cc0fb9adSAneesh Kumar K.V */ 4471cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 4472d33a1976SEric Sandeen grp_blk--; 4473d33a1976SEric Sandeen 4474bd86298eSLukas Czerner grp = ext4_get_group_number(sb, grp_blk); 4475c9de560dSAlex Tomas 4476c9de560dSAlex Tomas /* 4477c9de560dSAlex Tomas * possible race: 4478c9de560dSAlex Tomas * 4479c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 4480c9de560dSAlex Tomas * find block B in PA 4481c9de560dSAlex Tomas * copy on-disk bitmap to buddy 4482c9de560dSAlex Tomas * mark B in on-disk bitmap 4483c9de560dSAlex Tomas * drop PA from group 4484c9de560dSAlex Tomas * mark all PAs in buddy 4485c9de560dSAlex Tomas * 4486c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 4487c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 4488c9de560dSAlex Tomas * against that pair 4489c9de560dSAlex Tomas */ 4490c9de560dSAlex Tomas ext4_lock_group(sb, grp); 4491c9de560dSAlex Tomas list_del(&pa->pa_group_list); 4492c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 4493c9de560dSAlex Tomas 4494c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 4495c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4496c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 4497c9de560dSAlex Tomas 4498c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4499c9de560dSAlex Tomas } 4500c9de560dSAlex Tomas 4501c9de560dSAlex Tomas /* 4502c9de560dSAlex Tomas * creates new preallocated space for given inode 4503c9de560dSAlex Tomas */ 450453f86b17SRitesh Harjani static noinline_for_stack void 45054ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 4506c9de560dSAlex Tomas { 4507c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 450853accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 4509c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4510c9de560dSAlex Tomas struct ext4_group_info *grp; 4511c9de560dSAlex Tomas struct ext4_inode_info *ei; 4512c9de560dSAlex Tomas 4513c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 4514c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 4515c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 4516c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 451753f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 4518c9de560dSAlex Tomas 451953f86b17SRitesh Harjani pa = ac->ac_pa; 4520c9de560dSAlex Tomas 4521c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 4522c9de560dSAlex Tomas int winl; 4523c9de560dSAlex Tomas int wins; 4524c9de560dSAlex Tomas int win; 4525c9de560dSAlex Tomas int offs; 4526c9de560dSAlex Tomas 4527c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 4528c9de560dSAlex Tomas * so, found space must get proper lstart 4529c9de560dSAlex Tomas * to cover original request */ 4530c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 4531c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 4532c9de560dSAlex Tomas 4533c9de560dSAlex Tomas /* we're limited by original request in that 4534c9de560dSAlex Tomas * logical block must be covered any way 4535c9de560dSAlex Tomas * winl is window we can move our chunk within */ 4536c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 4537c9de560dSAlex Tomas 4538c9de560dSAlex Tomas /* also, we should cover whole original request */ 453953accfa9STheodore Ts'o wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); 4540c9de560dSAlex Tomas 4541c9de560dSAlex Tomas /* the smallest one defines real window */ 4542c9de560dSAlex Tomas win = min(winl, wins); 4543c9de560dSAlex Tomas 454453accfa9STheodore Ts'o offs = ac->ac_o_ex.fe_logical % 454553accfa9STheodore Ts'o EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 4546c9de560dSAlex Tomas if (offs && offs < win) 4547c9de560dSAlex Tomas win = offs; 4548c9de560dSAlex Tomas 454953accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - 4550810da240SLukas Czerner EXT4_NUM_B2C(sbi, win); 4551c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 4552c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 4553c9de560dSAlex Tomas } 4554c9de560dSAlex Tomas 4555c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 4556c9de560dSAlex Tomas * allocated blocks for history */ 4557c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 4558c9de560dSAlex Tomas 4559c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 4560c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4561c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 4562c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 4563c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 4564d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 4565d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 4566c9de560dSAlex Tomas pa->pa_deleted = 0; 4567cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 4568c9de560dSAlex Tomas 4569d3df1453SRitesh Harjani mb_debug(sb, "new inode pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 4570d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 45719bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 4572c9de560dSAlex Tomas 4573c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 457453accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 4575c9de560dSAlex Tomas 4576c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 4577c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 4578c9de560dSAlex Tomas 4579c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 4580c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 4581c9de560dSAlex Tomas 4582c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 4583c9de560dSAlex Tomas 4584c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 4585c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 4586c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 458727bc446eSbrookxu atomic_inc(&ei->i_prealloc_active); 4588c9de560dSAlex Tomas } 4589c9de560dSAlex Tomas 4590c9de560dSAlex Tomas /* 4591c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 4592c9de560dSAlex Tomas */ 459353f86b17SRitesh Harjani static noinline_for_stack void 45944ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 4595c9de560dSAlex Tomas { 4596c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 4597c9de560dSAlex Tomas struct ext4_locality_group *lg; 4598c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4599c9de560dSAlex Tomas struct ext4_group_info *grp; 4600c9de560dSAlex Tomas 4601c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 4602c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 4603c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 4604c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 460553f86b17SRitesh Harjani BUG_ON(ac->ac_pa == NULL); 4606c9de560dSAlex Tomas 460753f86b17SRitesh Harjani pa = ac->ac_pa; 4608c9de560dSAlex Tomas 4609c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 4610c9de560dSAlex Tomas * allocated blocks for history */ 4611c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 4612c9de560dSAlex Tomas 4613c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4614c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 4615c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 4616c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 4617c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 46186be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 4619d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 4620c9de560dSAlex Tomas pa->pa_deleted = 0; 4621cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 4622c9de560dSAlex Tomas 4623d3df1453SRitesh Harjani mb_debug(sb, "new group pa %p: %llu/%d for %u\n", pa, pa->pa_pstart, 4624d3df1453SRitesh Harjani pa->pa_len, pa->pa_lstart); 46259bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 4626c9de560dSAlex Tomas 4627c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 4628c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 4629c9de560dSAlex Tomas 4630c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 4631c9de560dSAlex Tomas lg = ac->ac_lg; 4632c9de560dSAlex Tomas BUG_ON(lg == NULL); 4633c9de560dSAlex Tomas 4634c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 4635c9de560dSAlex Tomas pa->pa_inode = NULL; 4636c9de560dSAlex Tomas 4637c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 4638c9de560dSAlex Tomas 46396be2ded1SAneesh Kumar K.V /* 46406be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 46416be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 46426be2ded1SAneesh Kumar K.V */ 4643c9de560dSAlex Tomas } 4644c9de560dSAlex Tomas 464553f86b17SRitesh Harjani static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 4646c9de560dSAlex Tomas { 4647c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 464853f86b17SRitesh Harjani ext4_mb_new_group_pa(ac); 4649c9de560dSAlex Tomas else 465053f86b17SRitesh Harjani ext4_mb_new_inode_pa(ac); 4651c9de560dSAlex Tomas } 4652c9de560dSAlex Tomas 4653c9de560dSAlex Tomas /* 4654c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 4655c9de560dSAlex Tomas * in-core bitmap and buddy. 4656c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 4657c9de560dSAlex Tomas * nobody else can find/use it. 4658c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 4659c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 4660c9de560dSAlex Tomas */ 46614ddfef7bSEric Sandeen static noinline_for_stack int 46624ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 46633e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 4664c9de560dSAlex Tomas { 4665c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4666c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4667498e5f24STheodore Ts'o unsigned int end; 4668498e5f24STheodore Ts'o unsigned int next; 4669c9de560dSAlex Tomas ext4_group_t group; 4670c9de560dSAlex Tomas ext4_grpblk_t bit; 4671ba80b101STheodore Ts'o unsigned long long grp_blk_start; 4672c9de560dSAlex Tomas int free = 0; 4673c9de560dSAlex Tomas 4674c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 4675c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 467653accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 4677c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 4678c9de560dSAlex Tomas end = bit + pa->pa_len; 4679c9de560dSAlex Tomas 4680c9de560dSAlex Tomas while (bit < end) { 4681ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 4682c9de560dSAlex Tomas if (bit >= end) 4683c9de560dSAlex Tomas break; 4684ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 4685d3df1453SRitesh Harjani mb_debug(sb, "free preallocated %u/%u in group %u\n", 46865a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 46875a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 4688c9de560dSAlex Tomas free += next - bit; 4689c9de560dSAlex Tomas 46903e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 469153accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 469253accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 4693a9c667f8SLukas Czerner next - bit); 4694c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 4695c9de560dSAlex Tomas bit = next + 1; 4696c9de560dSAlex Tomas } 4697c9de560dSAlex Tomas if (free != pa->pa_free) { 46989d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 469936bad423SRitesh Harjani "pa %p: logic %lu, phys. %lu, len %d", 4700c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 4701c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 470236bad423SRitesh Harjani pa->pa_len); 4703e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 470426346ff6SAneesh Kumar K.V free, pa->pa_free); 4705e56eb659SAneesh Kumar K.V /* 4706e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 4707e56eb659SAneesh Kumar K.V * from the bitmap and continue. 4708e56eb659SAneesh Kumar K.V */ 4709c9de560dSAlex Tomas } 4710c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 4711c9de560dSAlex Tomas 4712863c37fcSzhong jiang return 0; 4713c9de560dSAlex Tomas } 4714c9de560dSAlex Tomas 47154ddfef7bSEric Sandeen static noinline_for_stack int 47164ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 47173e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 4718c9de560dSAlex Tomas { 4719c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4720c9de560dSAlex Tomas ext4_group_t group; 4721c9de560dSAlex Tomas ext4_grpblk_t bit; 4722c9de560dSAlex Tomas 472360e07cf5SYongqiang Yang trace_ext4_mb_release_group_pa(sb, pa); 4724c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 4725c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 4726c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 4727c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 4728c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 47293e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 4730c9de560dSAlex Tomas 4731c9de560dSAlex Tomas return 0; 4732c9de560dSAlex Tomas } 4733c9de560dSAlex Tomas 4734c9de560dSAlex Tomas /* 4735c9de560dSAlex Tomas * releases all preallocations in given group 4736c9de560dSAlex Tomas * 4737c9de560dSAlex Tomas * first, we need to decide discard policy: 4738c9de560dSAlex Tomas * - when do we discard 4739c9de560dSAlex Tomas * 1) ENOSPC 4740c9de560dSAlex Tomas * - how many do we discard 4741c9de560dSAlex Tomas * 1) how many requested 4742c9de560dSAlex Tomas */ 47434ddfef7bSEric Sandeen static noinline_for_stack int 47444ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 4745c9de560dSAlex Tomas ext4_group_t group, int needed) 4746c9de560dSAlex Tomas { 4747c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 4748c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 4749c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 4750c9de560dSAlex Tomas struct list_head list; 4751c9de560dSAlex Tomas struct ext4_buddy e4b; 4752c9de560dSAlex Tomas int err; 4753c9de560dSAlex Tomas int busy = 0; 47545b3dc19dSJan Kara int free, free_total = 0; 4755c9de560dSAlex Tomas 4756d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for group %u\n", group); 4757c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 4758bbc4ec77SRitesh Harjani goto out_dbg; 4759c9de560dSAlex Tomas 4760574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 47619008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 47629008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 476354d3adbcSTheodore Ts'o ext4_error_err(sb, -err, 476454d3adbcSTheodore Ts'o "Error %d reading block bitmap for %u", 47659008a58eSDarrick J. Wong err, group); 4766bbc4ec77SRitesh Harjani goto out_dbg; 4767c9de560dSAlex Tomas } 4768c9de560dSAlex Tomas 4769c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 4770ce89f46cSAneesh Kumar K.V if (err) { 47719651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 47729651e6b2SKonstantin Khlebnikov err, group); 4773ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 4774bbc4ec77SRitesh Harjani goto out_dbg; 4775ce89f46cSAneesh Kumar K.V } 4776c9de560dSAlex Tomas 4777c9de560dSAlex Tomas if (needed == 0) 47787137d7a4STheodore Ts'o needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 4779c9de560dSAlex Tomas 4780c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4781c9de560dSAlex Tomas repeat: 47825b3dc19dSJan Kara free = 0; 4783c9de560dSAlex Tomas ext4_lock_group(sb, group); 4784c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 4785c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 4786c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4787c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4788c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4789c9de560dSAlex Tomas busy = 1; 4790c9de560dSAlex Tomas continue; 4791c9de560dSAlex Tomas } 4792c9de560dSAlex Tomas if (pa->pa_deleted) { 4793c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4794c9de560dSAlex Tomas continue; 4795c9de560dSAlex Tomas } 4796c9de560dSAlex Tomas 4797c9de560dSAlex Tomas /* seems this one can be freed ... */ 479827bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 4799c9de560dSAlex Tomas 480070022da8SYe Bin if (!free) 480170022da8SYe Bin this_cpu_inc(discard_pa_seq); 480270022da8SYe Bin 4803c9de560dSAlex Tomas /* we can trust pa_free ... */ 4804c9de560dSAlex Tomas free += pa->pa_free; 4805c9de560dSAlex Tomas 4806c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4807c9de560dSAlex Tomas 4808c9de560dSAlex Tomas list_del(&pa->pa_group_list); 4809c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 4810c9de560dSAlex Tomas } 4811c9de560dSAlex Tomas 4812c9de560dSAlex Tomas /* now free all selected PAs */ 4813c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4814c9de560dSAlex Tomas 4815c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 4816c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 4817c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4818c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 4819c9de560dSAlex Tomas 4820cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 48213e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 4822c9de560dSAlex Tomas else 48233e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4824c9de560dSAlex Tomas 4825c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4826c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4827c9de560dSAlex Tomas } 4828c9de560dSAlex Tomas 48295b3dc19dSJan Kara free_total += free; 48305b3dc19dSJan Kara 48315b3dc19dSJan Kara /* if we still need more blocks and some PAs were used, try again */ 48325b3dc19dSJan Kara if (free_total < needed && busy) { 48335b3dc19dSJan Kara ext4_unlock_group(sb, group); 48345b3dc19dSJan Kara cond_resched(); 48355b3dc19dSJan Kara busy = 0; 48365b3dc19dSJan Kara goto repeat; 48375b3dc19dSJan Kara } 4838c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4839e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4840c9de560dSAlex Tomas put_bh(bitmap_bh); 4841bbc4ec77SRitesh Harjani out_dbg: 4842d3df1453SRitesh Harjani mb_debug(sb, "discarded (%d) blocks preallocated for group %u bb_free (%d)\n", 48435b3dc19dSJan Kara free_total, group, grp->bb_free); 48445b3dc19dSJan Kara return free_total; 4845c9de560dSAlex Tomas } 4846c9de560dSAlex Tomas 4847c9de560dSAlex Tomas /* 4848c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 4849c9de560dSAlex Tomas * 4850c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 4851c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 4852c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 4853c9de560dSAlex Tomas * 4854c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 4855c9de560dSAlex Tomas */ 485627bc446eSbrookxu void ext4_discard_preallocations(struct inode *inode, unsigned int needed) 4857c9de560dSAlex Tomas { 4858c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 4859c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4860c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 4861c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 4862c9de560dSAlex Tomas ext4_group_t group = 0; 4863c9de560dSAlex Tomas struct list_head list; 4864c9de560dSAlex Tomas struct ext4_buddy e4b; 4865c9de560dSAlex Tomas int err; 4866c9de560dSAlex Tomas 4867c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 4868c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 4869c9de560dSAlex Tomas return; 4870c9de560dSAlex Tomas } 4871c9de560dSAlex Tomas 48728016e29fSHarshad Shirwadkar if (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) 48738016e29fSHarshad Shirwadkar return; 48748016e29fSHarshad Shirwadkar 4875d3df1453SRitesh Harjani mb_debug(sb, "discard preallocation for inode %lu\n", 4876d3df1453SRitesh Harjani inode->i_ino); 487727bc446eSbrookxu trace_ext4_discard_preallocations(inode, 487827bc446eSbrookxu atomic_read(&ei->i_prealloc_active), needed); 4879c9de560dSAlex Tomas 4880c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4881c9de560dSAlex Tomas 488227bc446eSbrookxu if (needed == 0) 488327bc446eSbrookxu needed = UINT_MAX; 488427bc446eSbrookxu 4885c9de560dSAlex Tomas repeat: 4886c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 4887c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 488827bc446eSbrookxu while (!list_empty(&ei->i_prealloc_list) && needed) { 488927bc446eSbrookxu pa = list_entry(ei->i_prealloc_list.prev, 4890c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 4891c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 4892c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4893c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4894c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 4895c9de560dSAlex Tomas * use preallocation while we're discarding it */ 4896c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4897c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 48989d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 48999d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 4900c9de560dSAlex Tomas WARN_ON(1); 4901c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4902c9de560dSAlex Tomas goto repeat; 4903c9de560dSAlex Tomas 4904c9de560dSAlex Tomas } 4905c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 490627bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 4907c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4908c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4909c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 491027bc446eSbrookxu needed--; 4911c9de560dSAlex Tomas continue; 4912c9de560dSAlex Tomas } 4913c9de560dSAlex Tomas 4914c9de560dSAlex Tomas /* someone is deleting pa right now */ 4915c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4916c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4917c9de560dSAlex Tomas 4918c9de560dSAlex Tomas /* we have to wait here because pa_deleted 4919c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 4920c9de560dSAlex Tomas * the list. as we might be called from 4921c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 4922c9de560dSAlex Tomas * and concurrent thread which is unlinking 4923c9de560dSAlex Tomas * pa from inode's list may access already 4924c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 4925c9de560dSAlex Tomas 4926c9de560dSAlex Tomas /* XXX: if this happens too often, we can 4927c9de560dSAlex Tomas * add a flag to force wait only in case 4928c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 4929c9de560dSAlex Tomas * regular truncate */ 4930c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4931c9de560dSAlex Tomas goto repeat; 4932c9de560dSAlex Tomas } 4933c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4934c9de560dSAlex Tomas 4935c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4936cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 4937bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 4938c9de560dSAlex Tomas 49399651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 49409651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 4941ce89f46cSAneesh Kumar K.V if (err) { 494254d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 49439651e6b2SKonstantin Khlebnikov err, group); 4944ce89f46cSAneesh Kumar K.V continue; 4945ce89f46cSAneesh Kumar K.V } 4946c9de560dSAlex Tomas 4947574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 49489008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 49499008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 495054d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d reading block bitmap for %u", 49519008a58eSDarrick J. Wong err, group); 4952e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4953ce89f46cSAneesh Kumar K.V continue; 4954c9de560dSAlex Tomas } 4955c9de560dSAlex Tomas 4956c9de560dSAlex Tomas ext4_lock_group(sb, group); 4957c9de560dSAlex Tomas list_del(&pa->pa_group_list); 49583e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4959c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4960c9de560dSAlex Tomas 4961e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4962c9de560dSAlex Tomas put_bh(bitmap_bh); 4963c9de560dSAlex Tomas 4964c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4965c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4966c9de560dSAlex Tomas } 4967c9de560dSAlex Tomas } 4968c9de560dSAlex Tomas 496953f86b17SRitesh Harjani static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac) 497053f86b17SRitesh Harjani { 497153f86b17SRitesh Harjani struct ext4_prealloc_space *pa; 497253f86b17SRitesh Harjani 497353f86b17SRitesh Harjani BUG_ON(ext4_pspace_cachep == NULL); 497453f86b17SRitesh Harjani pa = kmem_cache_zalloc(ext4_pspace_cachep, GFP_NOFS); 497553f86b17SRitesh Harjani if (!pa) 497653f86b17SRitesh Harjani return -ENOMEM; 497753f86b17SRitesh Harjani atomic_set(&pa->pa_count, 1); 497853f86b17SRitesh Harjani ac->ac_pa = pa; 497953f86b17SRitesh Harjani return 0; 498053f86b17SRitesh Harjani } 498153f86b17SRitesh Harjani 498253f86b17SRitesh Harjani static void ext4_mb_pa_free(struct ext4_allocation_context *ac) 498353f86b17SRitesh Harjani { 498453f86b17SRitesh Harjani struct ext4_prealloc_space *pa = ac->ac_pa; 498553f86b17SRitesh Harjani 498653f86b17SRitesh Harjani BUG_ON(!pa); 498753f86b17SRitesh Harjani ac->ac_pa = NULL; 498853f86b17SRitesh Harjani WARN_ON(!atomic_dec_and_test(&pa->pa_count)); 498953f86b17SRitesh Harjani kmem_cache_free(ext4_pspace_cachep, pa); 499053f86b17SRitesh Harjani } 499153f86b17SRitesh Harjani 49926ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 4993e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 4994c9de560dSAlex Tomas { 4995e68cf40cSRitesh Harjani ext4_group_t i, ngroups; 4996c9de560dSAlex Tomas 49979b5f6c9bSHarshad Shirwadkar if (ext4_test_mount_flag(sb, EXT4_MF_FS_ABORTED)) 4998e3570639SEric Sandeen return; 4999e3570639SEric Sandeen 50008df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 5001d3df1453SRitesh Harjani mb_debug(sb, "groups: "); 50028df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 5003c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 5004c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 5005c9de560dSAlex Tomas ext4_grpblk_t start; 5006c9de560dSAlex Tomas struct list_head *cur; 5007c9de560dSAlex Tomas ext4_lock_group(sb, i); 5008c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 5009c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 5010c9de560dSAlex Tomas pa_group_list); 5011c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 5012c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 5013c9de560dSAlex Tomas NULL, &start); 5014c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 5015d3df1453SRitesh Harjani mb_debug(sb, "PA:%u:%d:%d\n", i, start, 5016d3df1453SRitesh Harjani pa->pa_len); 5017c9de560dSAlex Tomas } 501860bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 5019d3df1453SRitesh Harjani mb_debug(sb, "%u: %d/%d\n", i, grp->bb_free, 5020d3df1453SRitesh Harjani grp->bb_fragments); 5021c9de560dSAlex Tomas } 5022c9de560dSAlex Tomas } 5023e68cf40cSRitesh Harjani 5024e68cf40cSRitesh Harjani static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 5025e68cf40cSRitesh Harjani { 5026e68cf40cSRitesh Harjani struct super_block *sb = ac->ac_sb; 5027e68cf40cSRitesh Harjani 50289b5f6c9bSHarshad Shirwadkar if (ext4_test_mount_flag(sb, EXT4_MF_FS_ABORTED)) 5029e68cf40cSRitesh Harjani return; 5030e68cf40cSRitesh Harjani 5031d3df1453SRitesh Harjani mb_debug(sb, "Can't allocate:" 5032e68cf40cSRitesh Harjani " Allocation context details:"); 5033d3df1453SRitesh Harjani mb_debug(sb, "status %u flags 0x%x", 5034e68cf40cSRitesh Harjani ac->ac_status, ac->ac_flags); 5035d3df1453SRitesh Harjani mb_debug(sb, "orig %lu/%lu/%lu@%lu, " 5036e68cf40cSRitesh Harjani "goal %lu/%lu/%lu@%lu, " 5037e68cf40cSRitesh Harjani "best %lu/%lu/%lu@%lu cr %d", 5038e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_group, 5039e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_start, 5040e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_len, 5041e68cf40cSRitesh Harjani (unsigned long)ac->ac_o_ex.fe_logical, 5042e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_group, 5043e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_start, 5044e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_len, 5045e68cf40cSRitesh Harjani (unsigned long)ac->ac_g_ex.fe_logical, 5046e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_group, 5047e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_start, 5048e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_len, 5049e68cf40cSRitesh Harjani (unsigned long)ac->ac_b_ex.fe_logical, 5050e68cf40cSRitesh Harjani (int)ac->ac_criteria); 5051d3df1453SRitesh Harjani mb_debug(sb, "%u found", ac->ac_found); 5052e68cf40cSRitesh Harjani ext4_mb_show_pa(sb); 5053e68cf40cSRitesh Harjani } 5054c9de560dSAlex Tomas #else 5055e68cf40cSRitesh Harjani static inline void ext4_mb_show_pa(struct super_block *sb) 5056e68cf40cSRitesh Harjani { 5057e68cf40cSRitesh Harjani return; 5058e68cf40cSRitesh Harjani } 5059c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 5060c9de560dSAlex Tomas { 5061e68cf40cSRitesh Harjani ext4_mb_show_pa(ac->ac_sb); 5062c9de560dSAlex Tomas return; 5063c9de560dSAlex Tomas } 5064c9de560dSAlex Tomas #endif 5065c9de560dSAlex Tomas 5066c9de560dSAlex Tomas /* 5067c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 5068c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 5069c9de560dSAlex Tomas * allocation which ever is larger 5070c9de560dSAlex Tomas * 5071b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 5072c9de560dSAlex Tomas */ 5073c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 5074c9de560dSAlex Tomas { 5075c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 5076c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 5077c9de560dSAlex Tomas loff_t size, isize; 5078c9de560dSAlex Tomas 5079c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 5080c9de560dSAlex Tomas return; 5081c9de560dSAlex Tomas 50824ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 50834ba74d00STheodore Ts'o return; 50844ba74d00STheodore Ts'o 508553accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 508650797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 508750797481STheodore Ts'o >> bsbits; 5088c9de560dSAlex Tomas 508982dd124cSNikolay Borisov if ((size == isize) && !ext4_fs_is_busy(sbi) && 509082dd124cSNikolay Borisov !inode_is_open_for_write(ac->ac_inode)) { 509150797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 509250797481STheodore Ts'o return; 509350797481STheodore Ts'o } 509450797481STheodore Ts'o 5095ebbe0277SRobin Dong if (sbi->s_mb_group_prealloc <= 0) { 5096ebbe0277SRobin Dong ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 5097ebbe0277SRobin Dong return; 5098ebbe0277SRobin Dong } 5099ebbe0277SRobin Dong 5100c9de560dSAlex Tomas /* don't use group allocation for large files */ 510171780577STheodore Ts'o size = max(size, isize); 5102cc483f10STao Ma if (size > sbi->s_mb_stream_request) { 51034ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 5104c9de560dSAlex Tomas return; 51054ba74d00STheodore Ts'o } 5106c9de560dSAlex Tomas 5107c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 5108c9de560dSAlex Tomas /* 5109c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 5110c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 5111c9de560dSAlex Tomas * request from multiple CPUs. 5112c9de560dSAlex Tomas */ 5113a0b6bc63SChristoph Lameter ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups); 5114c9de560dSAlex Tomas 5115c9de560dSAlex Tomas /* we're going to use group allocation */ 5116c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 5117c9de560dSAlex Tomas 5118c9de560dSAlex Tomas /* serialize all allocations in the group */ 5119c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 5120c9de560dSAlex Tomas } 5121c9de560dSAlex Tomas 51224ddfef7bSEric Sandeen static noinline_for_stack int 51234ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 5124c9de560dSAlex Tomas struct ext4_allocation_request *ar) 5125c9de560dSAlex Tomas { 5126c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 5127c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 5128c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 5129c9de560dSAlex Tomas ext4_group_t group; 5130498e5f24STheodore Ts'o unsigned int len; 5131498e5f24STheodore Ts'o ext4_fsblk_t goal; 5132c9de560dSAlex Tomas ext4_grpblk_t block; 5133c9de560dSAlex Tomas 5134c9de560dSAlex Tomas /* we can't allocate > group size */ 5135c9de560dSAlex Tomas len = ar->len; 5136c9de560dSAlex Tomas 5137c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 513840ae3487STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) 513940ae3487STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb); 5140c9de560dSAlex Tomas 5141c9de560dSAlex Tomas /* start searching from the goal */ 5142c9de560dSAlex Tomas goal = ar->goal; 5143c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 5144c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 5145c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 5146c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 5147c9de560dSAlex Tomas 5148c9de560dSAlex Tomas /* set up allocation goals */ 5149f5a44db5STheodore Ts'o ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); 5150c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 5151c9de560dSAlex Tomas ac->ac_sb = sb; 5152c9de560dSAlex Tomas ac->ac_inode = ar->inode; 515353accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 5154c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 5155c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 5156c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 515753accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 5158c9de560dSAlex Tomas ac->ac_flags = ar->flags; 5159c9de560dSAlex Tomas 51603cb77bd2Sbrookxu /* we have to define context: we'll work with a file or 5161c9de560dSAlex Tomas * locality group. this is a policy, actually */ 5162c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 5163c9de560dSAlex Tomas 5164d3df1453SRitesh Harjani mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, " 5165c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 5166c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 5167c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 5168c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 5169c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 517082dd124cSNikolay Borisov inode_is_open_for_write(ar->inode) ? "" : "non-"); 5171c9de560dSAlex Tomas return 0; 5172c9de560dSAlex Tomas 5173c9de560dSAlex Tomas } 5174c9de560dSAlex Tomas 51756be2ded1SAneesh Kumar K.V static noinline_for_stack void 51766be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 51776be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 51786be2ded1SAneesh Kumar K.V int order, int total_entries) 51796be2ded1SAneesh Kumar K.V { 51806be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 51816be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 51826be2ded1SAneesh Kumar K.V struct list_head discard_list; 51836be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 51846be2ded1SAneesh Kumar K.V 5185d3df1453SRitesh Harjani mb_debug(sb, "discard locality group preallocation\n"); 51866be2ded1SAneesh Kumar K.V 51876be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 51886be2ded1SAneesh Kumar K.V 51896be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 51906be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 519192e9c58cSMadhuparna Bhowmik pa_inode_list, 519292e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 51936be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 51946be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 51956be2ded1SAneesh Kumar K.V /* 51966be2ded1SAneesh Kumar K.V * This is the pa that we just used 51976be2ded1SAneesh Kumar K.V * for block allocation. So don't 51986be2ded1SAneesh Kumar K.V * free that 51996be2ded1SAneesh Kumar K.V */ 52006be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 52016be2ded1SAneesh Kumar K.V continue; 52026be2ded1SAneesh Kumar K.V } 52036be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 52046be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 52056be2ded1SAneesh Kumar K.V continue; 52066be2ded1SAneesh Kumar K.V } 52076be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 5208cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 52096be2ded1SAneesh Kumar K.V 52106be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 521127bc446eSbrookxu ext4_mb_mark_pa_deleted(sb, pa); 52126be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 52136be2ded1SAneesh Kumar K.V 52146be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 52156be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 52166be2ded1SAneesh Kumar K.V 52176be2ded1SAneesh Kumar K.V total_entries--; 52186be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 52196be2ded1SAneesh Kumar K.V /* 52206be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 52216be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 52226be2ded1SAneesh Kumar K.V * mak sure we don't call discard 52236be2ded1SAneesh Kumar K.V * soon for this list. 52246be2ded1SAneesh Kumar K.V */ 52256be2ded1SAneesh Kumar K.V break; 52266be2ded1SAneesh Kumar K.V } 52276be2ded1SAneesh Kumar K.V } 52286be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 52296be2ded1SAneesh Kumar K.V 52306be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 52319651e6b2SKonstantin Khlebnikov int err; 52326be2ded1SAneesh Kumar K.V 5233bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 52349651e6b2SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, group, &e4b, 52359651e6b2SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 52369651e6b2SKonstantin Khlebnikov if (err) { 523754d3adbcSTheodore Ts'o ext4_error_err(sb, -err, "Error %d loading buddy information for %u", 52389651e6b2SKonstantin Khlebnikov err, group); 52396be2ded1SAneesh Kumar K.V continue; 52406be2ded1SAneesh Kumar K.V } 52416be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 52426be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 52433e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 52446be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 52456be2ded1SAneesh Kumar K.V 5246e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 52476be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 52486be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 52496be2ded1SAneesh Kumar K.V } 52506be2ded1SAneesh Kumar K.V } 52516be2ded1SAneesh Kumar K.V 52526be2ded1SAneesh Kumar K.V /* 52536be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 52546be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 52556be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 52566be2ded1SAneesh Kumar K.V * 52576be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 52586be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 52596be2ded1SAneesh Kumar K.V */ 52606be2ded1SAneesh Kumar K.V 52616be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 52626be2ded1SAneesh Kumar K.V { 52636be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 52646be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 52656be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 52666be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 52676be2ded1SAneesh Kumar K.V 52686be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 52696be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 52706be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 52716be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 52726be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 5273f1167009SNiu Yawei spin_lock(&lg->lg_prealloc_lock); 52746be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 527592e9c58cSMadhuparna Bhowmik pa_inode_list, 527692e9c58cSMadhuparna Bhowmik lockdep_is_held(&lg->lg_prealloc_lock)) { 52776be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 52786be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 5279e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 52806be2ded1SAneesh Kumar K.V continue; 52816be2ded1SAneesh Kumar K.V } 52826be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 52836be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 52846be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 52856be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 52866be2ded1SAneesh Kumar K.V added = 1; 52876be2ded1SAneesh Kumar K.V /* 52886be2ded1SAneesh Kumar K.V * we want to count the total 52896be2ded1SAneesh Kumar K.V * number of entries in the list 52906be2ded1SAneesh Kumar K.V */ 52916be2ded1SAneesh Kumar K.V } 52926be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 52936be2ded1SAneesh Kumar K.V lg_prealloc_count++; 52946be2ded1SAneesh Kumar K.V } 52956be2ded1SAneesh Kumar K.V if (!added) 52966be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 52976be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 5298f1167009SNiu Yawei spin_unlock(&lg->lg_prealloc_lock); 52996be2ded1SAneesh Kumar K.V 53006be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 53016be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 53026be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 53036be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 53046be2ded1SAneesh Kumar K.V return; 53056be2ded1SAneesh Kumar K.V } 53066be2ded1SAneesh Kumar K.V return ; 53076be2ded1SAneesh Kumar K.V } 53086be2ded1SAneesh Kumar K.V 5309c9de560dSAlex Tomas /* 531027bc446eSbrookxu * if per-inode prealloc list is too long, trim some PA 531127bc446eSbrookxu */ 531227bc446eSbrookxu static void ext4_mb_trim_inode_pa(struct inode *inode) 531327bc446eSbrookxu { 531427bc446eSbrookxu struct ext4_inode_info *ei = EXT4_I(inode); 531527bc446eSbrookxu struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); 531627bc446eSbrookxu int count, delta; 531727bc446eSbrookxu 531827bc446eSbrookxu count = atomic_read(&ei->i_prealloc_active); 531927bc446eSbrookxu delta = (sbi->s_mb_max_inode_prealloc >> 2) + 1; 532027bc446eSbrookxu if (count > sbi->s_mb_max_inode_prealloc + delta) { 532127bc446eSbrookxu count -= sbi->s_mb_max_inode_prealloc; 532227bc446eSbrookxu ext4_discard_preallocations(inode, count); 532327bc446eSbrookxu } 532427bc446eSbrookxu } 532527bc446eSbrookxu 532627bc446eSbrookxu /* 5327c9de560dSAlex Tomas * release all resource we used in allocation 5328c9de560dSAlex Tomas */ 5329c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 5330c9de560dSAlex Tomas { 533127bc446eSbrookxu struct inode *inode = ac->ac_inode; 533227bc446eSbrookxu struct ext4_inode_info *ei = EXT4_I(inode); 533353accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 53346be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 53356be2ded1SAneesh Kumar K.V if (pa) { 5336cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 5337c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 53386be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 533953accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 534053accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 53416be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 53426be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 53436be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 534466d5e027Sbrookxu 53456be2ded1SAneesh Kumar K.V /* 53466be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 53476be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 53486be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 534944183d42SAmir Goldstein * doesn't grow big. 53506be2ded1SAneesh Kumar K.V */ 535166d5e027Sbrookxu if (likely(pa->pa_free)) { 53526be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 53536be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 53546be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 53556be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 5356c9de560dSAlex Tomas } 535766d5e027Sbrookxu } 535827bc446eSbrookxu 535927bc446eSbrookxu if (pa->pa_type == MB_INODE_PA) { 536027bc446eSbrookxu /* 536127bc446eSbrookxu * treat per-inode prealloc list as a lru list, then try 536227bc446eSbrookxu * to trim the least recently used PA. 536327bc446eSbrookxu */ 536427bc446eSbrookxu spin_lock(pa->pa_obj_lock); 536527bc446eSbrookxu list_move(&pa->pa_inode_list, &ei->i_prealloc_list); 536627bc446eSbrookxu spin_unlock(pa->pa_obj_lock); 536727bc446eSbrookxu } 536827bc446eSbrookxu 53696be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 5370c9de560dSAlex Tomas } 5371c9de560dSAlex Tomas if (ac->ac_bitmap_page) 537209cbfeafSKirill A. Shutemov put_page(ac->ac_bitmap_page); 5373c9de560dSAlex Tomas if (ac->ac_buddy_page) 537409cbfeafSKirill A. Shutemov put_page(ac->ac_buddy_page); 5375c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 5376c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 5377c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 537827bc446eSbrookxu ext4_mb_trim_inode_pa(inode); 5379c9de560dSAlex Tomas return 0; 5380c9de560dSAlex Tomas } 5381c9de560dSAlex Tomas 5382c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 5383c9de560dSAlex Tomas { 53848df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 5385c9de560dSAlex Tomas int ret; 5386c9de560dSAlex Tomas int freed = 0; 5387c9de560dSAlex Tomas 53889bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 53898df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 5390c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 5391c9de560dSAlex Tomas freed += ret; 5392c9de560dSAlex Tomas needed -= ret; 5393c9de560dSAlex Tomas } 5394c9de560dSAlex Tomas 5395c9de560dSAlex Tomas return freed; 5396c9de560dSAlex Tomas } 5397c9de560dSAlex Tomas 5398cf5e2ca6SRitesh Harjani static bool ext4_mb_discard_preallocations_should_retry(struct super_block *sb, 539907b5b8e1SRitesh Harjani struct ext4_allocation_context *ac, u64 *seq) 5400cf5e2ca6SRitesh Harjani { 5401cf5e2ca6SRitesh Harjani int freed; 540207b5b8e1SRitesh Harjani u64 seq_retry = 0; 540307b5b8e1SRitesh Harjani bool ret = false; 5404cf5e2ca6SRitesh Harjani 5405cf5e2ca6SRitesh Harjani freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 540607b5b8e1SRitesh Harjani if (freed) { 540707b5b8e1SRitesh Harjani ret = true; 540807b5b8e1SRitesh Harjani goto out_dbg; 540907b5b8e1SRitesh Harjani } 541007b5b8e1SRitesh Harjani seq_retry = ext4_get_discard_pa_seq_sum(); 541199377830SRitesh Harjani if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) { 541299377830SRitesh Harjani ac->ac_flags |= EXT4_MB_STRICT_CHECK; 541307b5b8e1SRitesh Harjani *seq = seq_retry; 541407b5b8e1SRitesh Harjani ret = true; 541507b5b8e1SRitesh Harjani } 541607b5b8e1SRitesh Harjani 541707b5b8e1SRitesh Harjani out_dbg: 541807b5b8e1SRitesh Harjani mb_debug(sb, "freed %d, retry ? %s\n", freed, ret ? "yes" : "no"); 541907b5b8e1SRitesh Harjani return ret; 5420cf5e2ca6SRitesh Harjani } 5421cf5e2ca6SRitesh Harjani 54228016e29fSHarshad Shirwadkar static ext4_fsblk_t ext4_mb_new_blocks_simple(handle_t *handle, 54238016e29fSHarshad Shirwadkar struct ext4_allocation_request *ar, int *errp); 54248016e29fSHarshad Shirwadkar 5425c9de560dSAlex Tomas /* 5426c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 5427c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 5428c9de560dSAlex Tomas * to usual allocation 5429c9de560dSAlex Tomas */ 5430c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 5431c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 5432c9de560dSAlex Tomas { 5433256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 5434c9de560dSAlex Tomas struct ext4_sb_info *sbi; 5435c9de560dSAlex Tomas struct super_block *sb; 5436c9de560dSAlex Tomas ext4_fsblk_t block = 0; 543760e58e0fSMingming Cao unsigned int inquota = 0; 543853accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 543907b5b8e1SRitesh Harjani u64 seq; 5440c9de560dSAlex Tomas 5441b10a44c3STheodore Ts'o might_sleep(); 5442c9de560dSAlex Tomas sb = ar->inode->i_sb; 5443c9de560dSAlex Tomas sbi = EXT4_SB(sb); 5444c9de560dSAlex Tomas 54459bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 54468016e29fSHarshad Shirwadkar if (sbi->s_mount_state & EXT4_FC_REPLAY) 54478016e29fSHarshad Shirwadkar return ext4_mb_new_blocks_simple(handle, ar, errp); 5448ba80b101STheodore Ts'o 544945dc63e7SDmitry Monakhov /* Allow to use superuser reservation for quota file */ 545002749a4cSTahsin Erdogan if (ext4_is_quota_file(ar->inode)) 545145dc63e7SDmitry Monakhov ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; 545245dc63e7SDmitry Monakhov 5453e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) { 545460e58e0fSMingming Cao /* Without delayed allocation we need to verify 545560e58e0fSMingming Cao * there is enough free blocks to do block allocation 545660e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 5457d2a17637SMingming Cao */ 545855f020dbSAllison Henderson while (ar->len && 5459e7d5f315STheodore Ts'o ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { 546055f020dbSAllison Henderson 5461030ba6bcSAneesh Kumar K.V /* let others to free the space */ 5462bb8b20edSLukas Czerner cond_resched(); 5463030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 5464030ba6bcSAneesh Kumar K.V } 5465030ba6bcSAneesh Kumar K.V if (!ar->len) { 5466bbc4ec77SRitesh Harjani ext4_mb_show_pa(sb); 546707031431SMingming Cao *errp = -ENOSPC; 546807031431SMingming Cao return 0; 546907031431SMingming Cao } 547053accfa9STheodore Ts'o reserv_clstrs = ar->len; 547155f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 547253accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 547353accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 547455f020dbSAllison Henderson } else { 547555f020dbSAllison Henderson while (ar->len && 547653accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 547753accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 547855f020dbSAllison Henderson 5479c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 5480c9de560dSAlex Tomas ar->len--; 5481c9de560dSAlex Tomas } 548255f020dbSAllison Henderson } 548360e58e0fSMingming Cao inquota = ar->len; 5484c9de560dSAlex Tomas if (ar->len == 0) { 5485c9de560dSAlex Tomas *errp = -EDQUOT; 54866c7a120aSAditya Kali goto out; 5487c9de560dSAlex Tomas } 548860e58e0fSMingming Cao } 5489d2a17637SMingming Cao 549085556c9aSWei Yongjun ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); 5491833576b3STheodore Ts'o if (!ac) { 5492363d4251SShen Feng ar->len = 0; 5493256bdb49SEric Sandeen *errp = -ENOMEM; 54946c7a120aSAditya Kali goto out; 5495256bdb49SEric Sandeen } 5496256bdb49SEric Sandeen 5497256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 5498c9de560dSAlex Tomas if (*errp) { 5499c9de560dSAlex Tomas ar->len = 0; 55006c7a120aSAditya Kali goto out; 5501c9de560dSAlex Tomas } 5502c9de560dSAlex Tomas 5503256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 550481198536SRitesh Harjani seq = this_cpu_read(discard_pa_seq); 5505256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 5506256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 5507256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 550853f86b17SRitesh Harjani 550953f86b17SRitesh Harjani *errp = ext4_mb_pa_alloc(ac); 551053f86b17SRitesh Harjani if (*errp) 551153f86b17SRitesh Harjani goto errout; 5512c9de560dSAlex Tomas repeat: 5513c9de560dSAlex Tomas /* allocate space in core */ 55146c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 551553f86b17SRitesh Harjani /* 551653f86b17SRitesh Harjani * pa allocated above is added to grp->bb_prealloc_list only 551753f86b17SRitesh Harjani * when we were able to allocate some block i.e. when 551853f86b17SRitesh Harjani * ac->ac_status == AC_STATUS_FOUND. 551953f86b17SRitesh Harjani * And error from above mean ac->ac_status != AC_STATUS_FOUND 552053f86b17SRitesh Harjani * So we have to free this pa here itself. 552153f86b17SRitesh Harjani */ 55222c00ef3eSAlexey Khoroshilov if (*errp) { 552353f86b17SRitesh Harjani ext4_mb_pa_free(ac); 55242c00ef3eSAlexey Khoroshilov ext4_discard_allocated_blocks(ac); 55252c00ef3eSAlexey Khoroshilov goto errout; 55262c00ef3eSAlexey Khoroshilov } 552753f86b17SRitesh Harjani if (ac->ac_status == AC_STATUS_FOUND && 552853f86b17SRitesh Harjani ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len) 552953f86b17SRitesh Harjani ext4_mb_pa_free(ac); 5530c9de560dSAlex Tomas } 5531256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 553253accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 5533554a5cccSVegard Nossum if (*errp) { 5534b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 55356d138cedSEric Sandeen goto errout; 55366d138cedSEric Sandeen } else { 5537256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 5538256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 5539519deca0SAneesh Kumar K.V } 5540c9de560dSAlex Tomas } else { 554107b5b8e1SRitesh Harjani if (ext4_mb_discard_preallocations_should_retry(sb, ac, &seq)) 5542c9de560dSAlex Tomas goto repeat; 554353f86b17SRitesh Harjani /* 554453f86b17SRitesh Harjani * If block allocation fails then the pa allocated above 554553f86b17SRitesh Harjani * needs to be freed here itself. 554653f86b17SRitesh Harjani */ 554753f86b17SRitesh Harjani ext4_mb_pa_free(ac); 5548c9de560dSAlex Tomas *errp = -ENOSPC; 55496c7a120aSAditya Kali } 55506c7a120aSAditya Kali 55516d138cedSEric Sandeen errout: 55526c7a120aSAditya Kali if (*errp) { 5553256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 5554c9de560dSAlex Tomas ar->len = 0; 5555256bdb49SEric Sandeen ext4_mb_show_ac(ac); 5556c9de560dSAlex Tomas } 5557256bdb49SEric Sandeen ext4_mb_release_context(ac); 55586c7a120aSAditya Kali out: 55596c7a120aSAditya Kali if (ac) 5560363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 556160e58e0fSMingming Cao if (inquota && ar->len < inquota) 556253accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 55630087d9fbSAneesh Kumar K.V if (!ar->len) { 5564e3cf5d5dSTheodore Ts'o if ((ar->flags & EXT4_MB_DELALLOC_RESERVED) == 0) 55650087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 556657042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 556753accfa9STheodore Ts'o reserv_clstrs); 55680087d9fbSAneesh Kumar K.V } 5569c9de560dSAlex Tomas 55709bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 5571ba80b101STheodore Ts'o 5572c9de560dSAlex Tomas return block; 5573c9de560dSAlex Tomas } 5574c9de560dSAlex Tomas 5575c894058dSAneesh Kumar K.V /* 5576c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 5577c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 5578c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 5579c894058dSAneesh Kumar K.V */ 5580a0154344SDaeho Jeong static void ext4_try_merge_freed_extent(struct ext4_sb_info *sbi, 5581a0154344SDaeho Jeong struct ext4_free_data *entry, 5582a0154344SDaeho Jeong struct ext4_free_data *new_entry, 5583a0154344SDaeho Jeong struct rb_root *entry_rb_root) 5584c894058dSAneesh Kumar K.V { 5585a0154344SDaeho Jeong if ((entry->efd_tid != new_entry->efd_tid) || 5586a0154344SDaeho Jeong (entry->efd_group != new_entry->efd_group)) 5587a0154344SDaeho Jeong return; 5588a0154344SDaeho Jeong if (entry->efd_start_cluster + entry->efd_count == 5589a0154344SDaeho Jeong new_entry->efd_start_cluster) { 5590a0154344SDaeho Jeong new_entry->efd_start_cluster = entry->efd_start_cluster; 5591a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 5592a0154344SDaeho Jeong } else if (new_entry->efd_start_cluster + new_entry->efd_count == 5593a0154344SDaeho Jeong entry->efd_start_cluster) { 5594a0154344SDaeho Jeong new_entry->efd_count += entry->efd_count; 5595a0154344SDaeho Jeong } else 5596a0154344SDaeho Jeong return; 5597a0154344SDaeho Jeong spin_lock(&sbi->s_md_lock); 5598a0154344SDaeho Jeong list_del(&entry->efd_list); 5599a0154344SDaeho Jeong spin_unlock(&sbi->s_md_lock); 5600a0154344SDaeho Jeong rb_erase(&entry->efd_node, entry_rb_root); 5601a0154344SDaeho Jeong kmem_cache_free(ext4_free_data_cachep, entry); 5602c894058dSAneesh Kumar K.V } 5603c894058dSAneesh Kumar K.V 56044ddfef7bSEric Sandeen static noinline_for_stack int 56054ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 56067a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 5607c9de560dSAlex Tomas { 5608e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 560984130193STheodore Ts'o ext4_grpblk_t cluster; 5610d08854f5STheodore Ts'o ext4_grpblk_t clusters = new_entry->efd_count; 56117a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 5612c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 5613c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 5614c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 5615c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 5616c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 5617c894058dSAneesh Kumar K.V 56180390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 5619c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 5620c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 5621c9de560dSAlex Tomas 562218aadd47SBobi Jam new_node = &new_entry->efd_node; 562318aadd47SBobi Jam cluster = new_entry->efd_start_cluster; 5624c9de560dSAlex Tomas 5625c894058dSAneesh Kumar K.V if (!*n) { 5626c894058dSAneesh Kumar K.V /* first free block exent. We need to 5627c894058dSAneesh Kumar K.V protect buddy cache from being freed, 5628c9de560dSAlex Tomas * otherwise we'll refresh it from 5629c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 5630c9de560dSAlex Tomas * blocks */ 563109cbfeafSKirill A. Shutemov get_page(e4b->bd_buddy_page); 563209cbfeafSKirill A. Shutemov get_page(e4b->bd_bitmap_page); 5633c894058dSAneesh Kumar K.V } 5634c894058dSAneesh Kumar K.V while (*n) { 5635c894058dSAneesh Kumar K.V parent = *n; 563618aadd47SBobi Jam entry = rb_entry(parent, struct ext4_free_data, efd_node); 563718aadd47SBobi Jam if (cluster < entry->efd_start_cluster) 5638c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 563918aadd47SBobi Jam else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) 5640c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 5641c894058dSAneesh Kumar K.V else { 5642e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 564384130193STheodore Ts'o ext4_group_first_block_no(sb, group) + 564484130193STheodore Ts'o EXT4_C2B(sbi, cluster), 5645e29136f8STheodore Ts'o "Block already on to-be-freed list"); 5646cca41553SChunguang Xu kmem_cache_free(ext4_free_data_cachep, new_entry); 5647c894058dSAneesh Kumar K.V return 0; 5648c9de560dSAlex Tomas } 5649c9de560dSAlex Tomas } 5650c9de560dSAlex Tomas 5651c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 5652c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 5653c894058dSAneesh Kumar K.V 5654c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 5655c894058dSAneesh Kumar K.V node = rb_prev(new_node); 5656c894058dSAneesh Kumar K.V if (node) { 565718aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 5658a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 5659a0154344SDaeho Jeong &(db->bb_free_root)); 5660c9de560dSAlex Tomas } 5661c894058dSAneesh Kumar K.V 5662c894058dSAneesh Kumar K.V node = rb_next(new_node); 5663c894058dSAneesh Kumar K.V if (node) { 566418aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 5665a0154344SDaeho Jeong ext4_try_merge_freed_extent(sbi, entry, new_entry, 5666a0154344SDaeho Jeong &(db->bb_free_root)); 5667c894058dSAneesh Kumar K.V } 5668a0154344SDaeho Jeong 5669d08854f5STheodore Ts'o spin_lock(&sbi->s_md_lock); 5670a0154344SDaeho Jeong list_add_tail(&new_entry->efd_list, &sbi->s_freed_data_list); 5671d08854f5STheodore Ts'o sbi->s_mb_free_pending += clusters; 5672d08854f5STheodore Ts'o spin_unlock(&sbi->s_md_lock); 5673c9de560dSAlex Tomas return 0; 5674c9de560dSAlex Tomas } 5675c9de560dSAlex Tomas 56768016e29fSHarshad Shirwadkar /* 56778016e29fSHarshad Shirwadkar * Simple allocator for Ext4 fast commit replay path. It searches for blocks 56788016e29fSHarshad Shirwadkar * linearly starting at the goal block and also excludes the blocks which 56798016e29fSHarshad Shirwadkar * are going to be in use after fast commit replay. 56808016e29fSHarshad Shirwadkar */ 56818016e29fSHarshad Shirwadkar static ext4_fsblk_t ext4_mb_new_blocks_simple(handle_t *handle, 56828016e29fSHarshad Shirwadkar struct ext4_allocation_request *ar, int *errp) 56838016e29fSHarshad Shirwadkar { 56848016e29fSHarshad Shirwadkar struct buffer_head *bitmap_bh; 56858016e29fSHarshad Shirwadkar struct super_block *sb = ar->inode->i_sb; 56868016e29fSHarshad Shirwadkar ext4_group_t group; 56878016e29fSHarshad Shirwadkar ext4_grpblk_t blkoff; 5688e121bd48SDan Carpenter int i = sb->s_blocksize; 56898016e29fSHarshad Shirwadkar ext4_fsblk_t goal, block; 56908016e29fSHarshad Shirwadkar struct ext4_super_block *es = EXT4_SB(sb)->s_es; 56918016e29fSHarshad Shirwadkar 56928016e29fSHarshad Shirwadkar goal = ar->goal; 56938016e29fSHarshad Shirwadkar if (goal < le32_to_cpu(es->s_first_data_block) || 56948016e29fSHarshad Shirwadkar goal >= ext4_blocks_count(es)) 56958016e29fSHarshad Shirwadkar goal = le32_to_cpu(es->s_first_data_block); 56968016e29fSHarshad Shirwadkar 56978016e29fSHarshad Shirwadkar ar->len = 0; 56988016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, goal, &group, &blkoff); 56998016e29fSHarshad Shirwadkar for (; group < ext4_get_groups_count(sb); group++) { 57008016e29fSHarshad Shirwadkar bitmap_bh = ext4_read_block_bitmap(sb, group); 57018016e29fSHarshad Shirwadkar if (IS_ERR(bitmap_bh)) { 57028016e29fSHarshad Shirwadkar *errp = PTR_ERR(bitmap_bh); 57038016e29fSHarshad Shirwadkar pr_warn("Failed to read block bitmap\n"); 57048016e29fSHarshad Shirwadkar return 0; 57058016e29fSHarshad Shirwadkar } 57068016e29fSHarshad Shirwadkar 57078016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, 57088016e29fSHarshad Shirwadkar max(ext4_group_first_block_no(sb, group), goal), 57098016e29fSHarshad Shirwadkar NULL, &blkoff); 57108016e29fSHarshad Shirwadkar i = mb_find_next_zero_bit(bitmap_bh->b_data, sb->s_blocksize, 57118016e29fSHarshad Shirwadkar blkoff); 57128016e29fSHarshad Shirwadkar brelse(bitmap_bh); 57138016e29fSHarshad Shirwadkar if (i >= sb->s_blocksize) 57148016e29fSHarshad Shirwadkar continue; 57158016e29fSHarshad Shirwadkar if (ext4_fc_replay_check_excluded(sb, 57168016e29fSHarshad Shirwadkar ext4_group_first_block_no(sb, group) + i)) 57178016e29fSHarshad Shirwadkar continue; 57188016e29fSHarshad Shirwadkar break; 57198016e29fSHarshad Shirwadkar } 57208016e29fSHarshad Shirwadkar 57218016e29fSHarshad Shirwadkar if (group >= ext4_get_groups_count(sb) && i >= sb->s_blocksize) 57228016e29fSHarshad Shirwadkar return 0; 57238016e29fSHarshad Shirwadkar 57248016e29fSHarshad Shirwadkar block = ext4_group_first_block_no(sb, group) + i; 57258016e29fSHarshad Shirwadkar ext4_mb_mark_bb(sb, block, 1, 1); 57268016e29fSHarshad Shirwadkar ar->len = 1; 57278016e29fSHarshad Shirwadkar 57288016e29fSHarshad Shirwadkar return block; 57298016e29fSHarshad Shirwadkar } 57308016e29fSHarshad Shirwadkar 57318016e29fSHarshad Shirwadkar static void ext4_free_blocks_simple(struct inode *inode, ext4_fsblk_t block, 57328016e29fSHarshad Shirwadkar unsigned long count) 57338016e29fSHarshad Shirwadkar { 57348016e29fSHarshad Shirwadkar struct buffer_head *bitmap_bh; 57358016e29fSHarshad Shirwadkar struct super_block *sb = inode->i_sb; 57368016e29fSHarshad Shirwadkar struct ext4_group_desc *gdp; 57378016e29fSHarshad Shirwadkar struct buffer_head *gdp_bh; 57388016e29fSHarshad Shirwadkar ext4_group_t group; 57398016e29fSHarshad Shirwadkar ext4_grpblk_t blkoff; 57408016e29fSHarshad Shirwadkar int already_freed = 0, err, i; 57418016e29fSHarshad Shirwadkar 57428016e29fSHarshad Shirwadkar ext4_get_group_no_and_offset(sb, block, &group, &blkoff); 57438016e29fSHarshad Shirwadkar bitmap_bh = ext4_read_block_bitmap(sb, group); 57448016e29fSHarshad Shirwadkar if (IS_ERR(bitmap_bh)) { 57458016e29fSHarshad Shirwadkar err = PTR_ERR(bitmap_bh); 57468016e29fSHarshad Shirwadkar pr_warn("Failed to read block bitmap\n"); 57478016e29fSHarshad Shirwadkar return; 57488016e29fSHarshad Shirwadkar } 57498016e29fSHarshad Shirwadkar gdp = ext4_get_group_desc(sb, group, &gdp_bh); 57508016e29fSHarshad Shirwadkar if (!gdp) 57518016e29fSHarshad Shirwadkar return; 57528016e29fSHarshad Shirwadkar 57538016e29fSHarshad Shirwadkar for (i = 0; i < count; i++) { 57548016e29fSHarshad Shirwadkar if (!mb_test_bit(blkoff + i, bitmap_bh->b_data)) 57558016e29fSHarshad Shirwadkar already_freed++; 57568016e29fSHarshad Shirwadkar } 57578016e29fSHarshad Shirwadkar mb_clear_bits(bitmap_bh->b_data, blkoff, count); 57588016e29fSHarshad Shirwadkar err = ext4_handle_dirty_metadata(NULL, NULL, bitmap_bh); 57598016e29fSHarshad Shirwadkar if (err) 57608016e29fSHarshad Shirwadkar return; 57618016e29fSHarshad Shirwadkar ext4_free_group_clusters_set( 57628016e29fSHarshad Shirwadkar sb, gdp, ext4_free_group_clusters(sb, gdp) + 57638016e29fSHarshad Shirwadkar count - already_freed); 57648016e29fSHarshad Shirwadkar ext4_block_bitmap_csum_set(sb, group, gdp, bitmap_bh); 57658016e29fSHarshad Shirwadkar ext4_group_desc_csum_set(sb, group, gdp); 57668016e29fSHarshad Shirwadkar ext4_handle_dirty_metadata(NULL, NULL, gdp_bh); 57678016e29fSHarshad Shirwadkar sync_dirty_buffer(bitmap_bh); 57688016e29fSHarshad Shirwadkar sync_dirty_buffer(gdp_bh); 57698016e29fSHarshad Shirwadkar brelse(bitmap_bh); 57708016e29fSHarshad Shirwadkar } 57718016e29fSHarshad Shirwadkar 577244338711STheodore Ts'o /** 577344338711STheodore Ts'o * ext4_free_blocks() -- Free given blocks and update quota 577444338711STheodore Ts'o * @handle: handle for this transaction 577544338711STheodore Ts'o * @inode: inode 5776c60990b3STheodore Ts'o * @bh: optional buffer of the block to be freed 5777c60990b3STheodore Ts'o * @block: starting physical block to be freed 5778c60990b3STheodore Ts'o * @count: number of blocks to be freed 57795def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 5780c9de560dSAlex Tomas */ 578144338711STheodore Ts'o void ext4_free_blocks(handle_t *handle, struct inode *inode, 5782e6362609STheodore Ts'o struct buffer_head *bh, ext4_fsblk_t block, 5783e6362609STheodore Ts'o unsigned long count, int flags) 5784c9de560dSAlex Tomas { 578526346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 5786c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 5787c9de560dSAlex Tomas struct ext4_group_desc *gdp; 5788498e5f24STheodore Ts'o unsigned int overflow; 5789c9de560dSAlex Tomas ext4_grpblk_t bit; 5790c9de560dSAlex Tomas struct buffer_head *gd_bh; 5791c9de560dSAlex Tomas ext4_group_t block_group; 5792c9de560dSAlex Tomas struct ext4_sb_info *sbi; 5793c9de560dSAlex Tomas struct ext4_buddy e4b; 579484130193STheodore Ts'o unsigned int count_clusters; 5795c9de560dSAlex Tomas int err = 0; 5796c9de560dSAlex Tomas int ret; 5797c9de560dSAlex Tomas 57988016e29fSHarshad Shirwadkar sbi = EXT4_SB(sb); 57998016e29fSHarshad Shirwadkar 58008016e29fSHarshad Shirwadkar if (sbi->s_mount_state & EXT4_FC_REPLAY) { 58018016e29fSHarshad Shirwadkar ext4_free_blocks_simple(inode, block, count); 58028016e29fSHarshad Shirwadkar return; 58038016e29fSHarshad Shirwadkar } 58048016e29fSHarshad Shirwadkar 5805b10a44c3STheodore Ts'o might_sleep(); 5806e6362609STheodore Ts'o if (bh) { 5807e6362609STheodore Ts'o if (block) 5808e6362609STheodore Ts'o BUG_ON(block != bh->b_blocknr); 5809e6362609STheodore Ts'o else 5810e6362609STheodore Ts'o block = bh->b_blocknr; 5811e6362609STheodore Ts'o } 5812c9de560dSAlex Tomas 58131f2acb60STheodore Ts'o if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 5814ce9f24ccSJan Kara !ext4_inode_block_valid(inode, block, count)) { 581512062dddSEric Sandeen ext4_error(sb, "Freeing blocks not in datazone - " 58160610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 5817c9de560dSAlex Tomas goto error_return; 5818c9de560dSAlex Tomas } 5819c9de560dSAlex Tomas 58200610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 5821e6362609STheodore Ts'o trace_ext4_free_blocks(inode, block, count, flags); 5822e6362609STheodore Ts'o 58239c02ac97SDaeho Jeong if (bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 58249c02ac97SDaeho Jeong BUG_ON(count > 1); 5825e6362609STheodore Ts'o 5826e6362609STheodore Ts'o ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 58279c02ac97SDaeho Jeong inode, bh, block); 5828e6362609STheodore Ts'o } 5829e6362609STheodore Ts'o 5830e6362609STheodore Ts'o /* 583184130193STheodore Ts'o * If the extent to be freed does not begin on a cluster 583284130193STheodore Ts'o * boundary, we need to deal with partial clusters at the 583384130193STheodore Ts'o * beginning and end of the extent. Normally we will free 583484130193STheodore Ts'o * blocks at the beginning or the end unless we are explicitly 583584130193STheodore Ts'o * requested to avoid doing so. 583684130193STheodore Ts'o */ 5837f5a44db5STheodore Ts'o overflow = EXT4_PBLK_COFF(sbi, block); 583884130193STheodore Ts'o if (overflow) { 583984130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { 584084130193STheodore Ts'o overflow = sbi->s_cluster_ratio - overflow; 584184130193STheodore Ts'o block += overflow; 584284130193STheodore Ts'o if (count > overflow) 584384130193STheodore Ts'o count -= overflow; 584484130193STheodore Ts'o else 584584130193STheodore Ts'o return; 584684130193STheodore Ts'o } else { 584784130193STheodore Ts'o block -= overflow; 584884130193STheodore Ts'o count += overflow; 584984130193STheodore Ts'o } 585084130193STheodore Ts'o } 5851f5a44db5STheodore Ts'o overflow = EXT4_LBLK_COFF(sbi, count); 585284130193STheodore Ts'o if (overflow) { 585384130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { 585484130193STheodore Ts'o if (count > overflow) 585584130193STheodore Ts'o count -= overflow; 585684130193STheodore Ts'o else 585784130193STheodore Ts'o return; 585884130193STheodore Ts'o } else 585984130193STheodore Ts'o count += sbi->s_cluster_ratio - overflow; 586084130193STheodore Ts'o } 586184130193STheodore Ts'o 58629c02ac97SDaeho Jeong if (!bh && (flags & EXT4_FREE_BLOCKS_FORGET)) { 58639c02ac97SDaeho Jeong int i; 5864f96c450dSDaeho Jeong int is_metadata = flags & EXT4_FREE_BLOCKS_METADATA; 58659c02ac97SDaeho Jeong 58669c02ac97SDaeho Jeong for (i = 0; i < count; i++) { 58679c02ac97SDaeho Jeong cond_resched(); 5868f96c450dSDaeho Jeong if (is_metadata) 58699c02ac97SDaeho Jeong bh = sb_find_get_block(inode->i_sb, block + i); 5870f96c450dSDaeho Jeong ext4_forget(handle, is_metadata, inode, bh, block + i); 58719c02ac97SDaeho Jeong } 58729c02ac97SDaeho Jeong } 58739c02ac97SDaeho Jeong 5874c9de560dSAlex Tomas do_more: 5875c9de560dSAlex Tomas overflow = 0; 5876c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 5877c9de560dSAlex Tomas 5878163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT( 5879163a203dSDarrick J. Wong ext4_get_group_info(sb, block_group)))) 5880163a203dSDarrick J. Wong return; 5881163a203dSDarrick J. Wong 5882c9de560dSAlex Tomas /* 5883c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 5884c9de560dSAlex Tomas * boundary. 5885c9de560dSAlex Tomas */ 588684130193STheodore Ts'o if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { 588784130193STheodore Ts'o overflow = EXT4_C2B(sbi, bit) + count - 588884130193STheodore Ts'o EXT4_BLOCKS_PER_GROUP(sb); 5889c9de560dSAlex Tomas count -= overflow; 5890c9de560dSAlex Tomas } 5891810da240SLukas Czerner count_clusters = EXT4_NUM_B2C(sbi, count); 5892574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 58939008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 58949008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 58959008a58eSDarrick J. Wong bitmap_bh = NULL; 5896c9de560dSAlex Tomas goto error_return; 5897ce89f46cSAneesh Kumar K.V } 5898c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 5899ce89f46cSAneesh Kumar K.V if (!gdp) { 5900ce89f46cSAneesh Kumar K.V err = -EIO; 5901c9de560dSAlex Tomas goto error_return; 5902ce89f46cSAneesh Kumar K.V } 5903c9de560dSAlex Tomas 5904c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 5905c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 5906c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 590749598e04SJun Piao sbi->s_itb_per_group) || 5908c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 590949598e04SJun Piao sbi->s_itb_per_group)) { 5910c9de560dSAlex Tomas 591112062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 59120610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 5913519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 5914519deca0SAneesh Kumar K.V goto error_return; 5915c9de560dSAlex Tomas } 5916c9de560dSAlex Tomas 5917c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 5918c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 5919c9de560dSAlex Tomas if (err) 5920c9de560dSAlex Tomas goto error_return; 5921c9de560dSAlex Tomas 5922c9de560dSAlex Tomas /* 5923c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 5924c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 5925c9de560dSAlex Tomas * using it 5926c9de560dSAlex Tomas */ 5927c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 5928c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 5929c9de560dSAlex Tomas if (err) 5930c9de560dSAlex Tomas goto error_return; 5931c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 5932c9de560dSAlex Tomas { 5933c9de560dSAlex Tomas int i; 593484130193STheodore Ts'o for (i = 0; i < count_clusters; i++) 5935c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 5936c9de560dSAlex Tomas } 5937c9de560dSAlex Tomas #endif 593884130193STheodore Ts'o trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); 5939c9de560dSAlex Tomas 5940adb7ef60SKonstantin Khlebnikov /* __GFP_NOFAIL: retry infinitely, ignore TIF_MEMDIE and memcg limit. */ 5941adb7ef60SKonstantin Khlebnikov err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b, 5942adb7ef60SKonstantin Khlebnikov GFP_NOFS|__GFP_NOFAIL); 5943920313a7SAneesh Kumar K.V if (err) 5944920313a7SAneesh Kumar K.V goto error_return; 5945e6362609STheodore Ts'o 5946f96c450dSDaeho Jeong /* 5947f96c450dSDaeho Jeong * We need to make sure we don't reuse the freed block until after the 5948f96c450dSDaeho Jeong * transaction is committed. We make an exception if the inode is to be 5949f96c450dSDaeho Jeong * written in writeback mode since writeback mode has weak data 5950f96c450dSDaeho Jeong * consistency guarantees. 5951f96c450dSDaeho Jeong */ 5952f96c450dSDaeho Jeong if (ext4_handle_valid(handle) && 5953f96c450dSDaeho Jeong ((flags & EXT4_FREE_BLOCKS_METADATA) || 5954f96c450dSDaeho Jeong !ext4_should_writeback_data(inode))) { 59557a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 59567a2fcbf7SAneesh Kumar K.V /* 59577444a072SMichal Hocko * We use __GFP_NOFAIL because ext4_free_blocks() is not allowed 59587444a072SMichal Hocko * to fail. 59597a2fcbf7SAneesh Kumar K.V */ 59607444a072SMichal Hocko new_entry = kmem_cache_alloc(ext4_free_data_cachep, 59617444a072SMichal Hocko GFP_NOFS|__GFP_NOFAIL); 596218aadd47SBobi Jam new_entry->efd_start_cluster = bit; 596318aadd47SBobi Jam new_entry->efd_group = block_group; 596418aadd47SBobi Jam new_entry->efd_count = count_clusters; 596518aadd47SBobi Jam new_entry->efd_tid = handle->h_transaction->t_tid; 5966955ce5f5SAneesh Kumar K.V 59677a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 596884130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 59697a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 5970c9de560dSAlex Tomas } else { 59717a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 59727a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 59737a2fcbf7SAneesh Kumar K.V * them with group lock_held 59747a2fcbf7SAneesh Kumar K.V */ 5975d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 5976a0154344SDaeho Jeong err = ext4_issue_discard(sb, block_group, bit, count, 5977a0154344SDaeho Jeong NULL); 5978d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 5979d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 5980d71c1ae2SLukas Czerner " group:%d block:%d count:%lu failed" 5981d71c1ae2SLukas Czerner " with %d", block_group, bit, count, 5982d71c1ae2SLukas Czerner err); 59838f9ff189SLukas Czerner } else 59848f9ff189SLukas Czerner EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); 5985d71c1ae2SLukas Czerner 5986955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 598784130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 598884130193STheodore Ts'o mb_free_blocks(inode, &e4b, bit, count_clusters); 5989c9de560dSAlex Tomas } 5990c9de560dSAlex Tomas 5991021b65bbSTheodore Ts'o ret = ext4_free_group_clusters(sb, gdp) + count_clusters; 5992021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, ret); 599379f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, gdp, bitmap_bh); 5994feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, gdp); 5995955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 5996c9de560dSAlex Tomas 5997772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 5998772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 599990ba983fSTheodore Ts'o atomic64_add(count_clusters, 60007c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 60017c990728SSuraj Jitindar Singh flex_group)->free_clusters); 6002772cb7c8SJose R. Santos } 6003772cb7c8SJose R. Santos 60049fe67149SEric Whitney /* 60059fe67149SEric Whitney * on a bigalloc file system, defer the s_freeclusters_counter 60069fe67149SEric Whitney * update to the caller (ext4_remove_space and friends) so they 60079fe67149SEric Whitney * can determine if a cluster freed here should be rereserved 60089fe67149SEric Whitney */ 60099fe67149SEric Whitney if (!(flags & EXT4_FREE_BLOCKS_RERESERVE_CLUSTER)) { 60107b415bf6SAditya Kali if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 60117b415bf6SAditya Kali dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); 60129fe67149SEric Whitney percpu_counter_add(&sbi->s_freeclusters_counter, 60139fe67149SEric Whitney count_clusters); 60149fe67149SEric Whitney } 60157d734532SJan Kara 60167d734532SJan Kara ext4_mb_unload_buddy(&e4b); 60177b415bf6SAditya Kali 60187a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 60197a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 60207a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 60217a2fcbf7SAneesh Kumar K.V 6022c9de560dSAlex Tomas /* And the group descriptor block */ 6023c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 60240390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 6025c9de560dSAlex Tomas if (!err) 6026c9de560dSAlex Tomas err = ret; 6027c9de560dSAlex Tomas 6028c9de560dSAlex Tomas if (overflow && !err) { 6029c9de560dSAlex Tomas block += count; 6030c9de560dSAlex Tomas count = overflow; 6031c9de560dSAlex Tomas put_bh(bitmap_bh); 6032c9de560dSAlex Tomas goto do_more; 6033c9de560dSAlex Tomas } 6034c9de560dSAlex Tomas error_return: 6035c9de560dSAlex Tomas brelse(bitmap_bh); 6036c9de560dSAlex Tomas ext4_std_error(sb, err); 6037c9de560dSAlex Tomas return; 6038c9de560dSAlex Tomas } 60397360d173SLukas Czerner 60407360d173SLukas Czerner /** 60410529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 60422846e820SAmir Goldstein * @handle: handle to this transaction 60432846e820SAmir Goldstein * @sb: super block 60444907cb7bSAnatol Pomozov * @block: start physical block to add to the block group 60452846e820SAmir Goldstein * @count: number of blocks to free 60462846e820SAmir Goldstein * 6047e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 60482846e820SAmir Goldstein */ 6049cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 60502846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 60512846e820SAmir Goldstein { 60522846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 60532846e820SAmir Goldstein struct buffer_head *gd_bh; 60542846e820SAmir Goldstein ext4_group_t block_group; 60552846e820SAmir Goldstein ext4_grpblk_t bit; 60562846e820SAmir Goldstein unsigned int i; 60572846e820SAmir Goldstein struct ext4_group_desc *desc; 60582846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 6059e73a347bSAmir Goldstein struct ext4_buddy e4b; 6060d77147ffSharshads int err = 0, ret, free_clusters_count; 6061d77147ffSharshads ext4_grpblk_t clusters_freed; 6062d77147ffSharshads ext4_fsblk_t first_cluster = EXT4_B2C(sbi, block); 6063d77147ffSharshads ext4_fsblk_t last_cluster = EXT4_B2C(sbi, block + count - 1); 6064d77147ffSharshads unsigned long cluster_count = last_cluster - first_cluster + 1; 60652846e820SAmir Goldstein 60662846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 60672846e820SAmir Goldstein 60684740b830SYongqiang Yang if (count == 0) 60694740b830SYongqiang Yang return 0; 60704740b830SYongqiang Yang 60712846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 60722846e820SAmir Goldstein /* 60732846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 60742846e820SAmir Goldstein * boundary. 60752846e820SAmir Goldstein */ 6076d77147ffSharshads if (bit + cluster_count > EXT4_CLUSTERS_PER_GROUP(sb)) { 6077d77147ffSharshads ext4_warning(sb, "too many blocks added to group %u", 6078cc7365dfSYongqiang Yang block_group); 6079cc7365dfSYongqiang Yang err = -EINVAL; 60802846e820SAmir Goldstein goto error_return; 6081cc7365dfSYongqiang Yang } 60822cd05cc3STheodore Ts'o 60832846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 60849008a58eSDarrick J. Wong if (IS_ERR(bitmap_bh)) { 60859008a58eSDarrick J. Wong err = PTR_ERR(bitmap_bh); 60869008a58eSDarrick J. Wong bitmap_bh = NULL; 60872846e820SAmir Goldstein goto error_return; 6088cc7365dfSYongqiang Yang } 6089cc7365dfSYongqiang Yang 60902846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 6091cc7365dfSYongqiang Yang if (!desc) { 6092cc7365dfSYongqiang Yang err = -EIO; 60932846e820SAmir Goldstein goto error_return; 6094cc7365dfSYongqiang Yang } 60952846e820SAmir Goldstein 60962846e820SAmir Goldstein if (in_range(ext4_block_bitmap(sb, desc), block, count) || 60972846e820SAmir Goldstein in_range(ext4_inode_bitmap(sb, desc), block, count) || 60982846e820SAmir Goldstein in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) || 60992846e820SAmir Goldstein in_range(block + count - 1, ext4_inode_table(sb, desc), 61002846e820SAmir Goldstein sbi->s_itb_per_group)) { 61012846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 61022846e820SAmir Goldstein "Block = %llu, count = %lu", 61032846e820SAmir Goldstein block, count); 6104cc7365dfSYongqiang Yang err = -EINVAL; 61052846e820SAmir Goldstein goto error_return; 61062846e820SAmir Goldstein } 61072846e820SAmir Goldstein 61082cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 61092cd05cc3STheodore Ts'o err = ext4_journal_get_write_access(handle, bitmap_bh); 61102846e820SAmir Goldstein if (err) 61112846e820SAmir Goldstein goto error_return; 61122846e820SAmir Goldstein 61132846e820SAmir Goldstein /* 61142846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 61152846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 61162846e820SAmir Goldstein * using it 61172846e820SAmir Goldstein */ 61182846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 61192846e820SAmir Goldstein err = ext4_journal_get_write_access(handle, gd_bh); 61202846e820SAmir Goldstein if (err) 61212846e820SAmir Goldstein goto error_return; 6122e73a347bSAmir Goldstein 6123d77147ffSharshads for (i = 0, clusters_freed = 0; i < cluster_count; i++) { 61242846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 6125e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 61262846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 61272846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 61282846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 61292846e820SAmir Goldstein } else { 6130d77147ffSharshads clusters_freed++; 61312846e820SAmir Goldstein } 61322846e820SAmir Goldstein } 6133e73a347bSAmir Goldstein 6134e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 6135e73a347bSAmir Goldstein if (err) 6136e73a347bSAmir Goldstein goto error_return; 6137e73a347bSAmir Goldstein 6138e73a347bSAmir Goldstein /* 6139e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 6140e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 6141e73a347bSAmir Goldstein * them with group lock_held 6142e73a347bSAmir Goldstein */ 61432846e820SAmir Goldstein ext4_lock_group(sb, block_group); 6144d77147ffSharshads mb_clear_bits(bitmap_bh->b_data, bit, cluster_count); 6145d77147ffSharshads mb_free_blocks(NULL, &e4b, bit, cluster_count); 6146d77147ffSharshads free_clusters_count = clusters_freed + 6147d77147ffSharshads ext4_free_group_clusters(sb, desc); 6148d77147ffSharshads ext4_free_group_clusters_set(sb, desc, free_clusters_count); 614979f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, desc, bitmap_bh); 6150feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, desc); 61512846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 615257042651STheodore Ts'o percpu_counter_add(&sbi->s_freeclusters_counter, 6153d77147ffSharshads clusters_freed); 61542846e820SAmir Goldstein 61552846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 61562846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 6157d77147ffSharshads atomic64_add(clusters_freed, 61587c990728SSuraj Jitindar Singh &sbi_array_rcu_deref(sbi, s_flex_groups, 61597c990728SSuraj Jitindar Singh flex_group)->free_clusters); 61602846e820SAmir Goldstein } 6161e73a347bSAmir Goldstein 6162e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 61632846e820SAmir Goldstein 61642846e820SAmir Goldstein /* We dirtied the bitmap block */ 61652846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 61662846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 61672846e820SAmir Goldstein 61682846e820SAmir Goldstein /* And the group descriptor block */ 61692846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 61702846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 61712846e820SAmir Goldstein if (!err) 61722846e820SAmir Goldstein err = ret; 61732846e820SAmir Goldstein 61742846e820SAmir Goldstein error_return: 61752846e820SAmir Goldstein brelse(bitmap_bh); 61762846e820SAmir Goldstein ext4_std_error(sb, err); 6177cc7365dfSYongqiang Yang return err; 61782846e820SAmir Goldstein } 61792846e820SAmir Goldstein 61802846e820SAmir Goldstein /** 61817360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 61827360d173SLukas Czerner * @sb: super block for the file system 61837360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 61847360d173SLukas Czerner * @count: number of blocks to TRIM 61857360d173SLukas Czerner * @group: alloc. group we are working with 61867360d173SLukas Czerner * @e4b: ext4 buddy for the group 61877360d173SLukas Czerner * 61887360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 61897360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 61907360d173SLukas Czerner * be called with under the group lock. 61917360d173SLukas Czerner */ 6192d71c1ae2SLukas Czerner static int ext4_trim_extent(struct super_block *sb, int start, int count, 61937360d173SLukas Czerner ext4_group_t group, struct ext4_buddy *e4b) 6194e2cbd587Sjon ernst __releases(bitlock) 6195e2cbd587Sjon ernst __acquires(bitlock) 61967360d173SLukas Czerner { 61977360d173SLukas Czerner struct ext4_free_extent ex; 6198d71c1ae2SLukas Czerner int ret = 0; 61997360d173SLukas Czerner 6200b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 6201b3d4c2b1STao Ma 62027360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 62037360d173SLukas Czerner 62047360d173SLukas Czerner ex.fe_start = start; 62057360d173SLukas Czerner ex.fe_group = group; 62067360d173SLukas Czerner ex.fe_len = count; 62077360d173SLukas Czerner 62087360d173SLukas Czerner /* 62097360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 62107360d173SLukas Czerner * being trimmed. 62117360d173SLukas Czerner */ 62127360d173SLukas Czerner mb_mark_used(e4b, &ex); 62137360d173SLukas Czerner ext4_unlock_group(sb, group); 6214a0154344SDaeho Jeong ret = ext4_issue_discard(sb, group, start, count, NULL); 62157360d173SLukas Czerner ext4_lock_group(sb, group); 62167360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 6217d71c1ae2SLukas Czerner return ret; 62187360d173SLukas Czerner } 62197360d173SLukas Czerner 62207360d173SLukas Czerner /** 62217360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 62227360d173SLukas Czerner * @sb: super block for file system 622322612283STao Ma * @group: group to be trimmed 62247360d173SLukas Czerner * @start: first group block to examine 62257360d173SLukas Czerner * @max: last group block to examine 62267360d173SLukas Czerner * @minblocks: minimum extent block count 62277360d173SLukas Czerner * 62287360d173SLukas Czerner * ext4_trim_all_free walks through group's buddy bitmap searching for free 62297360d173SLukas Czerner * extents. When the free block is found, ext4_trim_extent is called to TRIM 62307360d173SLukas Czerner * the extent. 62317360d173SLukas Czerner * 62327360d173SLukas Czerner * 62337360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 62347360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 62357360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 62367360d173SLukas Czerner * the group buddy bitmap. This is done until whole group is scanned. 62377360d173SLukas Czerner */ 62380b75a840SLukas Czerner static ext4_grpblk_t 623978944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 624078944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 624178944086SLukas Czerner ext4_grpblk_t minblocks) 62427360d173SLukas Czerner { 62437360d173SLukas Czerner void *bitmap; 6244169ddc3eSTao Ma ext4_grpblk_t next, count = 0, free_count = 0; 624578944086SLukas Czerner struct ext4_buddy e4b; 6246d71c1ae2SLukas Czerner int ret = 0; 62477360d173SLukas Czerner 6248b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 6249b3d4c2b1STao Ma 625078944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 625178944086SLukas Czerner if (ret) { 62529651e6b2SKonstantin Khlebnikov ext4_warning(sb, "Error %d loading buddy information for %u", 62539651e6b2SKonstantin Khlebnikov ret, group); 625478944086SLukas Czerner return ret; 625578944086SLukas Czerner } 625678944086SLukas Czerner bitmap = e4b.bd_bitmap; 625728739eeaSLukas Czerner 625828739eeaSLukas Czerner ext4_lock_group(sb, group); 62593d56b8d2STao Ma if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) && 62603d56b8d2STao Ma minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) 62613d56b8d2STao Ma goto out; 62623d56b8d2STao Ma 626378944086SLukas Czerner start = (e4b.bd_info->bb_first_free > start) ? 626478944086SLukas Czerner e4b.bd_info->bb_first_free : start; 62657360d173SLukas Czerner 6266913eed83SLukas Czerner while (start <= max) { 6267913eed83SLukas Czerner start = mb_find_next_zero_bit(bitmap, max + 1, start); 6268913eed83SLukas Czerner if (start > max) 62697360d173SLukas Czerner break; 6270913eed83SLukas Czerner next = mb_find_next_bit(bitmap, max + 1, start); 62717360d173SLukas Czerner 62727360d173SLukas Czerner if ((next - start) >= minblocks) { 6273d71c1ae2SLukas Czerner ret = ext4_trim_extent(sb, start, 627478944086SLukas Czerner next - start, group, &e4b); 6275d71c1ae2SLukas Czerner if (ret && ret != -EOPNOTSUPP) 6276d71c1ae2SLukas Czerner break; 6277d71c1ae2SLukas Czerner ret = 0; 62787360d173SLukas Czerner count += next - start; 62797360d173SLukas Czerner } 6280169ddc3eSTao Ma free_count += next - start; 62817360d173SLukas Czerner start = next + 1; 62827360d173SLukas Czerner 62837360d173SLukas Czerner if (fatal_signal_pending(current)) { 62847360d173SLukas Czerner count = -ERESTARTSYS; 62857360d173SLukas Czerner break; 62867360d173SLukas Czerner } 62877360d173SLukas Czerner 62887360d173SLukas Czerner if (need_resched()) { 62897360d173SLukas Czerner ext4_unlock_group(sb, group); 62907360d173SLukas Czerner cond_resched(); 62917360d173SLukas Czerner ext4_lock_group(sb, group); 62927360d173SLukas Czerner } 62937360d173SLukas Czerner 6294169ddc3eSTao Ma if ((e4b.bd_info->bb_free - free_count) < minblocks) 62957360d173SLukas Czerner break; 62967360d173SLukas Czerner } 62973d56b8d2STao Ma 6298d71c1ae2SLukas Czerner if (!ret) { 6299d71c1ae2SLukas Czerner ret = count; 63003d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 6301d71c1ae2SLukas Czerner } 63023d56b8d2STao Ma out: 63037360d173SLukas Czerner ext4_unlock_group(sb, group); 630478944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 63057360d173SLukas Czerner 63067360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 63077360d173SLukas Czerner count, group); 63087360d173SLukas Czerner 6309d71c1ae2SLukas Czerner return ret; 63107360d173SLukas Czerner } 63117360d173SLukas Czerner 63127360d173SLukas Czerner /** 63137360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 63147360d173SLukas Czerner * @sb: superblock for filesystem 63157360d173SLukas Czerner * @range: fstrim_range structure 63167360d173SLukas Czerner * 63177360d173SLukas Czerner * start: First Byte to trim 63187360d173SLukas Czerner * len: number of Bytes to trim from start 63197360d173SLukas Czerner * minlen: minimum extent length in Bytes 63207360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 63217360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 63227360d173SLukas Czerner * is invoked to trim all free space. 63237360d173SLukas Czerner */ 63247360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 63257360d173SLukas Czerner { 632678944086SLukas Czerner struct ext4_group_info *grp; 6327913eed83SLukas Czerner ext4_group_t group, first_group, last_group; 63287137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 6329913eed83SLukas Czerner uint64_t start, end, minlen, trimmed = 0; 63300f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 63310f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 6332913eed83SLukas Czerner ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); 63337360d173SLukas Czerner int ret = 0; 63347360d173SLukas Czerner 63357360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 6336913eed83SLukas Czerner end = start + (range->len >> sb->s_blocksize_bits) - 1; 6337aaf7d73eSLukas Czerner minlen = EXT4_NUM_B2C(EXT4_SB(sb), 6338aaf7d73eSLukas Czerner range->minlen >> sb->s_blocksize_bits); 63397360d173SLukas Czerner 63405de35e8dSLukas Czerner if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || 63415de35e8dSLukas Czerner start >= max_blks || 63425de35e8dSLukas Czerner range->len < sb->s_blocksize) 63437360d173SLukas Czerner return -EINVAL; 6344913eed83SLukas Czerner if (end >= max_blks) 6345913eed83SLukas Czerner end = max_blks - 1; 6346913eed83SLukas Czerner if (end <= first_data_blk) 634722f10457STao Ma goto out; 6348913eed83SLukas Czerner if (start < first_data_blk) 63490f0a25bfSJan Kara start = first_data_blk; 63507360d173SLukas Czerner 6351913eed83SLukas Czerner /* Determine first and last group to examine based on start and end */ 63527360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 63537137d7a4STheodore Ts'o &first_group, &first_cluster); 6354913eed83SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, 63557137d7a4STheodore Ts'o &last_group, &last_cluster); 63567360d173SLukas Czerner 6357913eed83SLukas Czerner /* end now represents the last cluster to discard in this group */ 6358913eed83SLukas Czerner end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 63597360d173SLukas Czerner 63607360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 636178944086SLukas Czerner grp = ext4_get_group_info(sb, group); 636278944086SLukas Czerner /* We only do this if the grp has never been initialized */ 636378944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 6364adb7ef60SKonstantin Khlebnikov ret = ext4_mb_init_group(sb, group, GFP_NOFS); 636578944086SLukas Czerner if (ret) 63667360d173SLukas Czerner break; 63677360d173SLukas Czerner } 63687360d173SLukas Czerner 63690ba08517STao Ma /* 6370913eed83SLukas Czerner * For all the groups except the last one, last cluster will 6371913eed83SLukas Czerner * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to 6372913eed83SLukas Czerner * change it for the last group, note that last_cluster is 6373913eed83SLukas Czerner * already computed earlier by ext4_get_group_no_and_offset() 63740ba08517STao Ma */ 6375913eed83SLukas Czerner if (group == last_group) 6376913eed83SLukas Czerner end = last_cluster; 63777360d173SLukas Czerner 637878944086SLukas Czerner if (grp->bb_free >= minlen) { 63797137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 6380913eed83SLukas Czerner end, minlen); 63817360d173SLukas Czerner if (cnt < 0) { 63827360d173SLukas Czerner ret = cnt; 63837360d173SLukas Czerner break; 63847360d173SLukas Czerner } 63857360d173SLukas Czerner trimmed += cnt; 638621e7fd22SLukas Czerner } 6387913eed83SLukas Czerner 6388913eed83SLukas Czerner /* 6389913eed83SLukas Czerner * For every group except the first one, we are sure 6390913eed83SLukas Czerner * that the first cluster to discard will be cluster #0. 6391913eed83SLukas Czerner */ 63927137d7a4STheodore Ts'o first_cluster = 0; 63937360d173SLukas Czerner } 63947360d173SLukas Czerner 63953d56b8d2STao Ma if (!ret) 63963d56b8d2STao Ma atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); 63973d56b8d2STao Ma 639822f10457STao Ma out: 6399aaf7d73eSLukas Czerner range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; 64007360d173SLukas Czerner return ret; 64017360d173SLukas Czerner } 64020c9ec4beSDarrick J. Wong 64030c9ec4beSDarrick J. Wong /* Iterate all the free extents in the group. */ 64040c9ec4beSDarrick J. Wong int 64050c9ec4beSDarrick J. Wong ext4_mballoc_query_range( 64060c9ec4beSDarrick J. Wong struct super_block *sb, 64070c9ec4beSDarrick J. Wong ext4_group_t group, 64080c9ec4beSDarrick J. Wong ext4_grpblk_t start, 64090c9ec4beSDarrick J. Wong ext4_grpblk_t end, 64100c9ec4beSDarrick J. Wong ext4_mballoc_query_range_fn formatter, 64110c9ec4beSDarrick J. Wong void *priv) 64120c9ec4beSDarrick J. Wong { 64130c9ec4beSDarrick J. Wong void *bitmap; 64140c9ec4beSDarrick J. Wong ext4_grpblk_t next; 64150c9ec4beSDarrick J. Wong struct ext4_buddy e4b; 64160c9ec4beSDarrick J. Wong int error; 64170c9ec4beSDarrick J. Wong 64180c9ec4beSDarrick J. Wong error = ext4_mb_load_buddy(sb, group, &e4b); 64190c9ec4beSDarrick J. Wong if (error) 64200c9ec4beSDarrick J. Wong return error; 64210c9ec4beSDarrick J. Wong bitmap = e4b.bd_bitmap; 64220c9ec4beSDarrick J. Wong 64230c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 64240c9ec4beSDarrick J. Wong 64250c9ec4beSDarrick J. Wong start = (e4b.bd_info->bb_first_free > start) ? 64260c9ec4beSDarrick J. Wong e4b.bd_info->bb_first_free : start; 64270c9ec4beSDarrick J. Wong if (end >= EXT4_CLUSTERS_PER_GROUP(sb)) 64280c9ec4beSDarrick J. Wong end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 64290c9ec4beSDarrick J. Wong 64300c9ec4beSDarrick J. Wong while (start <= end) { 64310c9ec4beSDarrick J. Wong start = mb_find_next_zero_bit(bitmap, end + 1, start); 64320c9ec4beSDarrick J. Wong if (start > end) 64330c9ec4beSDarrick J. Wong break; 64340c9ec4beSDarrick J. Wong next = mb_find_next_bit(bitmap, end + 1, start); 64350c9ec4beSDarrick J. Wong 64360c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 64370c9ec4beSDarrick J. Wong error = formatter(sb, group, start, next - start, priv); 64380c9ec4beSDarrick J. Wong if (error) 64390c9ec4beSDarrick J. Wong goto out_unload; 64400c9ec4beSDarrick J. Wong ext4_lock_group(sb, group); 64410c9ec4beSDarrick J. Wong 64420c9ec4beSDarrick J. Wong start = next + 1; 64430c9ec4beSDarrick J. Wong } 64440c9ec4beSDarrick J. Wong 64450c9ec4beSDarrick J. Wong ext4_unlock_group(sb, group); 64460c9ec4beSDarrick J. Wong out_unload: 64470c9ec4beSDarrick J. Wong ext4_mb_unload_buddy(&e4b); 64480c9ec4beSDarrick J. Wong 64490c9ec4beSDarrick J. Wong return error; 64500c9ec4beSDarrick J. Wong } 6451