1c9de560dSAlex Tomas /* 2c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 3c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 4c9de560dSAlex Tomas * 5c9de560dSAlex Tomas * This program is free software; you can redistribute it and/or modify 6c9de560dSAlex Tomas * it under the terms of the GNU General Public License version 2 as 7c9de560dSAlex Tomas * published by the Free Software Foundation. 8c9de560dSAlex Tomas * 9c9de560dSAlex Tomas * This program is distributed in the hope that it will be useful, 10c9de560dSAlex Tomas * but WITHOUT ANY WARRANTY; without even the implied warranty of 11c9de560dSAlex Tomas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12c9de560dSAlex Tomas * GNU General Public License for more details. 13c9de560dSAlex Tomas * 14c9de560dSAlex Tomas * You should have received a copy of the GNU General Public Licens 15c9de560dSAlex Tomas * along with this program; if not, write to the Free Software 16c9de560dSAlex Tomas * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- 17c9de560dSAlex Tomas */ 18c9de560dSAlex Tomas 19c9de560dSAlex Tomas 20c9de560dSAlex Tomas /* 21c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 22c9de560dSAlex Tomas */ 23c9de560dSAlex Tomas 2418aadd47SBobi Jam #include "ext4_jbd2.h" 258f6e39a7SMingming Cao #include "mballoc.h" 2628623c2fSTheodore Ts'o #include <linux/log2.h> 27a0b30c12STheodore Ts'o #include <linux/module.h> 285a0e3ad6STejun Heo #include <linux/slab.h> 299bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 309bffad1eSTheodore Ts'o 31a0b30c12STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 32a0b30c12STheodore Ts'o ushort ext4_mballoc_debug __read_mostly; 33a0b30c12STheodore Ts'o 34a0b30c12STheodore Ts'o module_param_named(mballoc_debug, ext4_mballoc_debug, ushort, 0644); 35a0b30c12STheodore Ts'o MODULE_PARM_DESC(mballoc_debug, "Debugging level for ext4's mballoc"); 36a0b30c12STheodore Ts'o #endif 37a0b30c12STheodore Ts'o 38c9de560dSAlex Tomas /* 39c9de560dSAlex Tomas * MUSTDO: 40c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 41c9de560dSAlex Tomas * - search for metadata in few groups 42c9de560dSAlex Tomas * 43c9de560dSAlex Tomas * TODO v4: 44c9de560dSAlex Tomas * - normalization should take into account whether file is still open 45c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 46c9de560dSAlex Tomas * - don't normalize tails 47c9de560dSAlex Tomas * - quota 48c9de560dSAlex Tomas * - reservation for superuser 49c9de560dSAlex Tomas * 50c9de560dSAlex Tomas * TODO v3: 51c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 52c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 53c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 54c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 55c9de560dSAlex Tomas * - error handling 56c9de560dSAlex Tomas */ 57c9de560dSAlex Tomas 58c9de560dSAlex Tomas /* 59c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 60c9de560dSAlex Tomas * near to the goal(block) value specified. 61c9de560dSAlex Tomas * 62b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 63b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 64b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 65b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 66b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 67b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 68b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 69b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 70b713a5ecSTheodore Ts'o * terms of number of blocks. 71c9de560dSAlex Tomas * 72c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 73b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 74c9de560dSAlex Tomas * 75b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 76b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 77b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 78b713a5ecSTheodore Ts'o * represented as: 79c9de560dSAlex Tomas * 80c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 81c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 8253accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 8353accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 84c9de560dSAlex Tomas * 85c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 86c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 87caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 88caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 89c9de560dSAlex Tomas * 90c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 91c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 92c9de560dSAlex Tomas * pa_free. 93c9de560dSAlex Tomas * 94c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 95c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 96caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 97c9de560dSAlex Tomas * 98c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 99c9de560dSAlex Tomas * 100c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 101c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 102c9de560dSAlex Tomas * 103c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 10425985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 105c9de560dSAlex Tomas * 106c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 107c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 108c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 109c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 110c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 111c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 112c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 113c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 114c9de560dSAlex Tomas * inode as: 115c9de560dSAlex Tomas * 116c9de560dSAlex Tomas * { page } 117c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 118c9de560dSAlex Tomas * 119c9de560dSAlex Tomas * 120c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 121c9de560dSAlex Tomas * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE / 122c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 123c9de560dSAlex Tomas * which is blocks_per_page/2 124c9de560dSAlex Tomas * 125c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 126c9de560dSAlex Tomas * away when the filesystem is unmounted. 127c9de560dSAlex Tomas * 128c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 129c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 130c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 131c9de560dSAlex Tomas * 132c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 133c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 134c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 135c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 136c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 137c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 13827baebb8STheodore Ts'o * sbi->s_mb_group_prealloc. The default value of s_mb_group_prealloc is 13927baebb8STheodore Ts'o * dependent on the cluster size; for non-bigalloc file systems, it is 140c9de560dSAlex Tomas * 512 blocks. This can be tuned via 141d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 142c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 143c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 144d7a1fee1SDan Ehrenberg * the smallest multiple of the stripe value (sbi->s_stripe) which is 145d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 146c9de560dSAlex Tomas * 147d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 148c9de560dSAlex Tomas * 149b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 150b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 151b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 152c9de560dSAlex Tomas * 153b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 154c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 155c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 156b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 157af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 158b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 159b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 160b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 161c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 162b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 163c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 164c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 165c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 166caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 167c9de560dSAlex Tomas * checked. 168c9de560dSAlex Tomas * 169c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 170c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 171c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 172c9de560dSAlex Tomas * subsequent request. 173c9de560dSAlex Tomas */ 174c9de560dSAlex Tomas 175c9de560dSAlex Tomas /* 176c9de560dSAlex Tomas * mballoc operates on the following data: 177c9de560dSAlex Tomas * - on-disk bitmap 178c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 179c9de560dSAlex Tomas * - preallocation descriptors (PAs) 180c9de560dSAlex Tomas * 181c9de560dSAlex Tomas * there are two types of preallocations: 182c9de560dSAlex Tomas * - inode 183c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 184c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 185c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 186c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 187c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 188c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 189c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 190c9de560dSAlex Tomas * must discard all preallocated blocks. 191c9de560dSAlex Tomas * - locality group 192c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 193c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 194c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 195c9de560dSAlex Tomas * it's consumed from the beginning to the end. 196c9de560dSAlex Tomas * 197c9de560dSAlex Tomas * relation between them can be expressed as: 198c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 199c9de560dSAlex Tomas * 200c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 201c9de560dSAlex Tomas * - allocated blocks (persistent) 202c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 203c9de560dSAlex Tomas * 204c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 205c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 206c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 207c9de560dSAlex Tomas * 208c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 209c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 210c9de560dSAlex Tomas * 211c9de560dSAlex Tomas * all operations can be expressed as: 212c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 213c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 214c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 215c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 216c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 217c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 218c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 219c9de560dSAlex Tomas * is used in real operation because we can't know actual used 220c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 221c9de560dSAlex Tomas * 222c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 223c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 224c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 225c9de560dSAlex Tomas * the following knowledge: 226c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 227c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 228c9de560dSAlex Tomas * nobody can re-allocate that block 229c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 230c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 231c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 232c9de560dSAlex Tomas * block 233c9de560dSAlex Tomas * 234c9de560dSAlex Tomas * so, now we're building a concurrency table: 235c9de560dSAlex Tomas * - init buddy vs. 236c9de560dSAlex Tomas * - new PA 237c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 238c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 239c9de560dSAlex Tomas * - use inode PA 240c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 241c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 242c9de560dSAlex Tomas * - discard inode PA 243c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 244c9de560dSAlex Tomas * - use locality group PA 245c9de560dSAlex Tomas * again PA-=N must be serialized with init 246c9de560dSAlex Tomas * - discard locality group PA 247c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 248c9de560dSAlex Tomas * - new PA vs. 249c9de560dSAlex Tomas * - use inode PA 250c9de560dSAlex Tomas * i_data_sem serializes them 251c9de560dSAlex Tomas * - discard inode PA 252c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 253c9de560dSAlex Tomas * - use locality group PA 254c9de560dSAlex Tomas * some mutex should serialize them 255c9de560dSAlex Tomas * - discard locality group PA 256c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 257c9de560dSAlex Tomas * - use inode PA 258c9de560dSAlex Tomas * - use inode PA 259c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 260c9de560dSAlex Tomas * - discard inode PA 261c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 262c9de560dSAlex Tomas * - use locality group PA 263c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 264c9de560dSAlex Tomas * - discard locality group PA 265c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 266c9de560dSAlex Tomas * 267c9de560dSAlex Tomas * now we're ready to make few consequences: 268c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 269c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 270c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 271c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 272c9de560dSAlex Tomas * any discard or they're serialized somehow 273c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 274c9de560dSAlex Tomas * 275c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 276c9de560dSAlex Tomas * in this case, but we should care about concurrent init 277c9de560dSAlex Tomas * 278c9de560dSAlex Tomas */ 279c9de560dSAlex Tomas 280c9de560dSAlex Tomas /* 281c9de560dSAlex Tomas * Logic in few words: 282c9de560dSAlex Tomas * 283c9de560dSAlex Tomas * - allocation: 284c9de560dSAlex Tomas * load group 285c9de560dSAlex Tomas * find blocks 286c9de560dSAlex Tomas * mark bits in on-disk bitmap 287c9de560dSAlex Tomas * release group 288c9de560dSAlex Tomas * 289c9de560dSAlex Tomas * - use preallocation: 290c9de560dSAlex Tomas * find proper PA (per-inode or group) 291c9de560dSAlex Tomas * load group 292c9de560dSAlex Tomas * mark bits in on-disk bitmap 293c9de560dSAlex Tomas * release group 294c9de560dSAlex Tomas * release PA 295c9de560dSAlex Tomas * 296c9de560dSAlex Tomas * - free: 297c9de560dSAlex Tomas * load group 298c9de560dSAlex Tomas * mark bits in on-disk bitmap 299c9de560dSAlex Tomas * release group 300c9de560dSAlex Tomas * 301c9de560dSAlex Tomas * - discard preallocations in group: 302c9de560dSAlex Tomas * mark PAs deleted 303c9de560dSAlex Tomas * move them onto local list 304c9de560dSAlex Tomas * load on-disk bitmap 305c9de560dSAlex Tomas * load group 306c9de560dSAlex Tomas * remove PA from object (inode or locality group) 307c9de560dSAlex Tomas * mark free blocks in-core 308c9de560dSAlex Tomas * 309c9de560dSAlex Tomas * - discard inode's preallocations: 310c9de560dSAlex Tomas */ 311c9de560dSAlex Tomas 312c9de560dSAlex Tomas /* 313c9de560dSAlex Tomas * Locking rules 314c9de560dSAlex Tomas * 315c9de560dSAlex Tomas * Locks: 316c9de560dSAlex Tomas * - bitlock on a group (group) 317c9de560dSAlex Tomas * - object (inode/locality) (object) 318c9de560dSAlex Tomas * - per-pa lock (pa) 319c9de560dSAlex Tomas * 320c9de560dSAlex Tomas * Paths: 321c9de560dSAlex Tomas * - new pa 322c9de560dSAlex Tomas * object 323c9de560dSAlex Tomas * group 324c9de560dSAlex Tomas * 325c9de560dSAlex Tomas * - find and use pa: 326c9de560dSAlex Tomas * pa 327c9de560dSAlex Tomas * 328c9de560dSAlex Tomas * - release consumed pa: 329c9de560dSAlex Tomas * pa 330c9de560dSAlex Tomas * group 331c9de560dSAlex Tomas * object 332c9de560dSAlex Tomas * 333c9de560dSAlex Tomas * - generate in-core bitmap: 334c9de560dSAlex Tomas * group 335c9de560dSAlex Tomas * pa 336c9de560dSAlex Tomas * 337c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 338c9de560dSAlex Tomas * object 339c9de560dSAlex Tomas * pa 340c9de560dSAlex Tomas * group 341c9de560dSAlex Tomas * 342c9de560dSAlex Tomas * - discard all for given group: 343c9de560dSAlex Tomas * group 344c9de560dSAlex Tomas * pa 345c9de560dSAlex Tomas * group 346c9de560dSAlex Tomas * object 347c9de560dSAlex Tomas * 348c9de560dSAlex Tomas */ 349c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 350c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 35118aadd47SBobi Jam static struct kmem_cache *ext4_free_data_cachep; 352fb1813f4SCurt Wohlgemuth 353fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 354fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 355fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 3562892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 357fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 358fb1813f4SCurt Wohlgemuth 3592892c15dSEric Sandeen static const char *ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 3602892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 3612892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 3622892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 3632892c15dSEric Sandeen }; 3642892c15dSEric Sandeen 365c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 366c3a326a6SAneesh Kumar K.V ext4_group_t group); 3677a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 3687a2fcbf7SAneesh Kumar K.V ext4_group_t group); 36918aadd47SBobi Jam static void ext4_free_data_callback(struct super_block *sb, 37018aadd47SBobi Jam struct ext4_journal_cb_entry *jce, int rc); 371c3a326a6SAneesh Kumar K.V 372ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 373ffad0a44SAneesh Kumar K.V { 374c9de560dSAlex Tomas #if BITS_PER_LONG == 64 375ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 376ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 377c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 378ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 379ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 380c9de560dSAlex Tomas #else 381c9de560dSAlex Tomas #error "how many bits you are?!" 382c9de560dSAlex Tomas #endif 383ffad0a44SAneesh Kumar K.V return addr; 384ffad0a44SAneesh Kumar K.V } 385c9de560dSAlex Tomas 386c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 387c9de560dSAlex Tomas { 388c9de560dSAlex Tomas /* 389c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 390c9de560dSAlex Tomas * needs unsigned long aligned address 391c9de560dSAlex Tomas */ 392ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 393c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 394c9de560dSAlex Tomas } 395c9de560dSAlex Tomas 396c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 397c9de560dSAlex Tomas { 398ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 399c9de560dSAlex Tomas ext4_set_bit(bit, addr); 400c9de560dSAlex Tomas } 401c9de560dSAlex Tomas 402c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 403c9de560dSAlex Tomas { 404ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 405c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 406c9de560dSAlex Tomas } 407c9de560dSAlex Tomas 408eabe0444SAndrey Sidorov static inline int mb_test_and_clear_bit(int bit, void *addr) 409eabe0444SAndrey Sidorov { 410eabe0444SAndrey Sidorov addr = mb_correct_addr_and_bit(&bit, addr); 411eabe0444SAndrey Sidorov return ext4_test_and_clear_bit(bit, addr); 412eabe0444SAndrey Sidorov } 413eabe0444SAndrey Sidorov 414ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 415ffad0a44SAneesh Kumar K.V { 416e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 417ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 418e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 419ffad0a44SAneesh Kumar K.V start += fix; 420ffad0a44SAneesh Kumar K.V 421e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 422e7dfb246SAneesh Kumar K.V if (ret > max) 423e7dfb246SAneesh Kumar K.V return max; 424e7dfb246SAneesh Kumar K.V return ret; 425ffad0a44SAneesh Kumar K.V } 426ffad0a44SAneesh Kumar K.V 427ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 428ffad0a44SAneesh Kumar K.V { 429e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 430ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 431e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 432ffad0a44SAneesh Kumar K.V start += fix; 433ffad0a44SAneesh Kumar K.V 434e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 435e7dfb246SAneesh Kumar K.V if (ret > max) 436e7dfb246SAneesh Kumar K.V return max; 437e7dfb246SAneesh Kumar K.V return ret; 438ffad0a44SAneesh Kumar K.V } 439ffad0a44SAneesh Kumar K.V 440c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 441c9de560dSAlex Tomas { 442c9de560dSAlex Tomas char *bb; 443c9de560dSAlex Tomas 444c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 445c9de560dSAlex Tomas BUG_ON(max == NULL); 446c9de560dSAlex Tomas 447c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 448c9de560dSAlex Tomas *max = 0; 449c9de560dSAlex Tomas return NULL; 450c9de560dSAlex Tomas } 451c9de560dSAlex Tomas 452c9de560dSAlex Tomas /* at order 0 we see each particular block */ 45384b775a3SColy Li if (order == 0) { 454c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 455c5e8f3f3STheodore Ts'o return e4b->bd_bitmap; 45684b775a3SColy Li } 457c9de560dSAlex Tomas 458c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 459c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 460c9de560dSAlex Tomas 461c9de560dSAlex Tomas return bb; 462c9de560dSAlex Tomas } 463c9de560dSAlex Tomas 464c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 465c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 466c9de560dSAlex Tomas int first, int count) 467c9de560dSAlex Tomas { 468c9de560dSAlex Tomas int i; 469c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 470c9de560dSAlex Tomas 471c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 472c9de560dSAlex Tomas return; 473bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 474c9de560dSAlex Tomas for (i = 0; i < count; i++) { 475c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 476c9de560dSAlex Tomas ext4_fsblk_t blocknr; 4775661bd68SAkinobu Mita 4785661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 47953accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 4805d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 481e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 482e29136f8STheodore Ts'o blocknr, 483e29136f8STheodore Ts'o "freeing block already freed " 484e29136f8STheodore Ts'o "(bit %u)", 485e29136f8STheodore Ts'o first + i); 486c9de560dSAlex Tomas } 487c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 488c9de560dSAlex Tomas } 489c9de560dSAlex Tomas } 490c9de560dSAlex Tomas 491c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 492c9de560dSAlex Tomas { 493c9de560dSAlex Tomas int i; 494c9de560dSAlex Tomas 495c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 496c9de560dSAlex Tomas return; 497bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 498c9de560dSAlex Tomas for (i = 0; i < count; i++) { 499c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 500c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 501c9de560dSAlex Tomas } 502c9de560dSAlex Tomas } 503c9de560dSAlex Tomas 504c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 505c9de560dSAlex Tomas { 506c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 507c9de560dSAlex Tomas unsigned char *b1, *b2; 508c9de560dSAlex Tomas int i; 509c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 510c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 511c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 512c9de560dSAlex Tomas if (b1[i] != b2[i]) { 5139d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 5149d8b9ec4STheodore Ts'o "corruption in group %u " 5154776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 5169d8b9ec4STheodore Ts'o "on disk/prealloc", 517c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 518c9de560dSAlex Tomas BUG(); 519c9de560dSAlex Tomas } 520c9de560dSAlex Tomas } 521c9de560dSAlex Tomas } 522c9de560dSAlex Tomas } 523c9de560dSAlex Tomas 524c9de560dSAlex Tomas #else 525c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 526c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 527c9de560dSAlex Tomas { 528c9de560dSAlex Tomas return; 529c9de560dSAlex Tomas } 530c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 531c9de560dSAlex Tomas int first, int count) 532c9de560dSAlex Tomas { 533c9de560dSAlex Tomas return; 534c9de560dSAlex Tomas } 535c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 536c9de560dSAlex Tomas { 537c9de560dSAlex Tomas return; 538c9de560dSAlex Tomas } 539c9de560dSAlex Tomas #endif 540c9de560dSAlex Tomas 541c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 542c9de560dSAlex Tomas 543c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 544c9de560dSAlex Tomas do { \ 545c9de560dSAlex Tomas if (!(assert)) { \ 546c9de560dSAlex Tomas printk(KERN_EMERG \ 547c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 548c9de560dSAlex Tomas function, file, line, # assert); \ 549c9de560dSAlex Tomas BUG(); \ 550c9de560dSAlex Tomas } \ 551c9de560dSAlex Tomas } while (0) 552c9de560dSAlex Tomas 553c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 554c9de560dSAlex Tomas const char *function, int line) 555c9de560dSAlex Tomas { 556c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 557c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 558c9de560dSAlex Tomas int max; 559c9de560dSAlex Tomas int max2; 560c9de560dSAlex Tomas int i; 561c9de560dSAlex Tomas int j; 562c9de560dSAlex Tomas int k; 563c9de560dSAlex Tomas int count; 564c9de560dSAlex Tomas struct ext4_group_info *grp; 565c9de560dSAlex Tomas int fragments = 0; 566c9de560dSAlex Tomas int fstart; 567c9de560dSAlex Tomas struct list_head *cur; 568c9de560dSAlex Tomas void *buddy; 569c9de560dSAlex Tomas void *buddy2; 570c9de560dSAlex Tomas 571c9de560dSAlex Tomas { 572c9de560dSAlex Tomas static int mb_check_counter; 573c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 574c9de560dSAlex Tomas return 0; 575c9de560dSAlex Tomas } 576c9de560dSAlex Tomas 577c9de560dSAlex Tomas while (order > 1) { 578c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 579c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 580c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 581c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 582c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 583c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 584c9de560dSAlex Tomas 585c9de560dSAlex Tomas count = 0; 586c9de560dSAlex Tomas for (i = 0; i < max; i++) { 587c9de560dSAlex Tomas 588c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 589c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 590c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 591c9de560dSAlex Tomas MB_CHECK_ASSERT( 592c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 593c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 594c9de560dSAlex Tomas MB_CHECK_ASSERT( 595c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 596c9de560dSAlex Tomas } 597c9de560dSAlex Tomas continue; 598c9de560dSAlex Tomas } 599c9de560dSAlex Tomas 6000a10da73SRobin Dong /* both bits in buddy2 must be 1 */ 601c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 602c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 603c9de560dSAlex Tomas 604c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 605c9de560dSAlex Tomas k = (i * (1 << order)) + j; 606c9de560dSAlex Tomas MB_CHECK_ASSERT( 607c5e8f3f3STheodore Ts'o !mb_test_bit(k, e4b->bd_bitmap)); 608c9de560dSAlex Tomas } 609c9de560dSAlex Tomas count++; 610c9de560dSAlex Tomas } 611c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 612c9de560dSAlex Tomas order--; 613c9de560dSAlex Tomas } 614c9de560dSAlex Tomas 615c9de560dSAlex Tomas fstart = -1; 616c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 617c9de560dSAlex Tomas for (i = 0; i < max; i++) { 618c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 619c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 620c9de560dSAlex Tomas if (fstart == -1) { 621c9de560dSAlex Tomas fragments++; 622c9de560dSAlex Tomas fstart = i; 623c9de560dSAlex Tomas } 624c9de560dSAlex Tomas continue; 625c9de560dSAlex Tomas } 626c9de560dSAlex Tomas fstart = -1; 627c9de560dSAlex Tomas /* check used bits only */ 628c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 629c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 630c9de560dSAlex Tomas k = i >> j; 631c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 632c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 633c9de560dSAlex Tomas } 634c9de560dSAlex Tomas } 635c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 636c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 637c9de560dSAlex Tomas 638c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 639c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 640c9de560dSAlex Tomas ext4_group_t groupnr; 641c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 64260bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 64360bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 644c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 64560bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 646c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 647c9de560dSAlex Tomas } 648c9de560dSAlex Tomas return 0; 649c9de560dSAlex Tomas } 650c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 651c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 65246e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 653c9de560dSAlex Tomas #else 654c9de560dSAlex Tomas #define mb_check_buddy(e4b) 655c9de560dSAlex Tomas #endif 656c9de560dSAlex Tomas 6577c786059SColy Li /* 6587c786059SColy Li * Divide blocks started from @first with length @len into 6597c786059SColy Li * smaller chunks with power of 2 blocks. 6607c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 6617c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 6627c786059SColy Li */ 663c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 664a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 665c9de560dSAlex Tomas struct ext4_group_info *grp) 666c9de560dSAlex Tomas { 667c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 668a36b4498SEric Sandeen ext4_grpblk_t min; 669a36b4498SEric Sandeen ext4_grpblk_t max; 670a36b4498SEric Sandeen ext4_grpblk_t chunk; 671c9de560dSAlex Tomas unsigned short border; 672c9de560dSAlex Tomas 6737137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 674c9de560dSAlex Tomas 675c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 676c9de560dSAlex Tomas 677c9de560dSAlex Tomas while (len > 0) { 678c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 679c9de560dSAlex Tomas max = ffs(first | border) - 1; 680c9de560dSAlex Tomas 681c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 682c9de560dSAlex Tomas min = fls(len) - 1; 683c9de560dSAlex Tomas 684c9de560dSAlex Tomas if (max < min) 685c9de560dSAlex Tomas min = max; 686c9de560dSAlex Tomas chunk = 1 << min; 687c9de560dSAlex Tomas 688c9de560dSAlex Tomas /* mark multiblock chunks only */ 689c9de560dSAlex Tomas grp->bb_counters[min]++; 690c9de560dSAlex Tomas if (min > 0) 691c9de560dSAlex Tomas mb_clear_bit(first >> min, 692c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 693c9de560dSAlex Tomas 694c9de560dSAlex Tomas len -= chunk; 695c9de560dSAlex Tomas first += chunk; 696c9de560dSAlex Tomas } 697c9de560dSAlex Tomas } 698c9de560dSAlex Tomas 6998a57d9d6SCurt Wohlgemuth /* 7008a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 7018a57d9d6SCurt Wohlgemuth * group. 7028a57d9d6SCurt Wohlgemuth */ 7038a57d9d6SCurt Wohlgemuth static void 7048a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 7058a57d9d6SCurt Wohlgemuth { 7068a57d9d6SCurt Wohlgemuth int i; 7078a57d9d6SCurt Wohlgemuth int bits; 7088a57d9d6SCurt Wohlgemuth 7098a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = -1; /* uninit */ 7108a57d9d6SCurt Wohlgemuth 7118a57d9d6SCurt Wohlgemuth bits = sb->s_blocksize_bits + 1; 7128a57d9d6SCurt Wohlgemuth for (i = bits; i >= 0; i--) { 7138a57d9d6SCurt Wohlgemuth if (grp->bb_counters[i] > 0) { 7148a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 7158a57d9d6SCurt Wohlgemuth break; 7168a57d9d6SCurt Wohlgemuth } 7178a57d9d6SCurt Wohlgemuth } 7188a57d9d6SCurt Wohlgemuth } 7198a57d9d6SCurt Wohlgemuth 720089ceeccSEric Sandeen static noinline_for_stack 721089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 722c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 723c9de560dSAlex Tomas { 724c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 7257137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 726a36b4498SEric Sandeen ext4_grpblk_t i = 0; 727a36b4498SEric Sandeen ext4_grpblk_t first; 728a36b4498SEric Sandeen ext4_grpblk_t len; 729c9de560dSAlex Tomas unsigned free = 0; 730c9de560dSAlex Tomas unsigned fragments = 0; 731c9de560dSAlex Tomas unsigned long long period = get_cycles(); 732c9de560dSAlex Tomas 733c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 734c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 735ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 736c9de560dSAlex Tomas grp->bb_first_free = i; 737c9de560dSAlex Tomas while (i < max) { 738c9de560dSAlex Tomas fragments++; 739c9de560dSAlex Tomas first = i; 740ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 741c9de560dSAlex Tomas len = i - first; 742c9de560dSAlex Tomas free += len; 743c9de560dSAlex Tomas if (len > 1) 744c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 745c9de560dSAlex Tomas else 746c9de560dSAlex Tomas grp->bb_counters[0]++; 747c9de560dSAlex Tomas if (i < max) 748ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 749c9de560dSAlex Tomas } 750c9de560dSAlex Tomas grp->bb_fragments = fragments; 751c9de560dSAlex Tomas 752c9de560dSAlex Tomas if (free != grp->bb_free) { 753e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 754163a203dSDarrick J. Wong "%u clusters in bitmap, %u in gd; " 755163a203dSDarrick J. Wong "block bitmap corrupt.", 756e29136f8STheodore Ts'o free, grp->bb_free); 757e56eb659SAneesh Kumar K.V /* 758163a203dSDarrick J. Wong * If we intend to continue, we consider group descriptor 759e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 760e56eb659SAneesh Kumar K.V */ 761c9de560dSAlex Tomas grp->bb_free = free; 762163a203dSDarrick J. Wong set_bit(EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT, &grp->bb_state); 763c9de560dSAlex Tomas } 7648a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 765c9de560dSAlex Tomas 766c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 767c9de560dSAlex Tomas 768c9de560dSAlex Tomas period = get_cycles() - period; 769c9de560dSAlex Tomas spin_lock(&EXT4_SB(sb)->s_bal_lock); 770c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_buddies_generated++; 771c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_generation_time += period; 772c9de560dSAlex Tomas spin_unlock(&EXT4_SB(sb)->s_bal_lock); 773c9de560dSAlex Tomas } 774c9de560dSAlex Tomas 775eabe0444SAndrey Sidorov static void mb_regenerate_buddy(struct ext4_buddy *e4b) 776eabe0444SAndrey Sidorov { 777eabe0444SAndrey Sidorov int count; 778eabe0444SAndrey Sidorov int order = 1; 779eabe0444SAndrey Sidorov void *buddy; 780eabe0444SAndrey Sidorov 781eabe0444SAndrey Sidorov while ((buddy = mb_find_buddy(e4b, order++, &count))) { 782eabe0444SAndrey Sidorov ext4_set_bits(buddy, 0, count); 783eabe0444SAndrey Sidorov } 784eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments = 0; 785eabe0444SAndrey Sidorov memset(e4b->bd_info->bb_counters, 0, 786eabe0444SAndrey Sidorov sizeof(*e4b->bd_info->bb_counters) * 787eabe0444SAndrey Sidorov (e4b->bd_sb->s_blocksize_bits + 2)); 788eabe0444SAndrey Sidorov 789eabe0444SAndrey Sidorov ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy, 790eabe0444SAndrey Sidorov e4b->bd_bitmap, e4b->bd_group); 791eabe0444SAndrey Sidorov } 792eabe0444SAndrey Sidorov 793c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 794c9de560dSAlex Tomas * for convenience. The information regarding each group 795c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 796c9de560dSAlex Tomas * block bitmap and buddy information. The information are 797c9de560dSAlex Tomas * stored in the inode as 798c9de560dSAlex Tomas * 799c9de560dSAlex Tomas * { page } 800c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 801c9de560dSAlex Tomas * 802c9de560dSAlex Tomas * 803c9de560dSAlex Tomas * one block each for bitmap and buddy information. 804c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 805c9de560dSAlex Tomas * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks. 806c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 807c9de560dSAlex Tomas * is blocks_per_page/2 8088a57d9d6SCurt Wohlgemuth * 8098a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 8108a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 811c9de560dSAlex Tomas */ 812c9de560dSAlex Tomas 813c9de560dSAlex Tomas static int ext4_mb_init_cache(struct page *page, char *incore) 814c9de560dSAlex Tomas { 8158df9675fSTheodore Ts'o ext4_group_t ngroups; 816c9de560dSAlex Tomas int blocksize; 817c9de560dSAlex Tomas int blocks_per_page; 818c9de560dSAlex Tomas int groups_per_page; 819c9de560dSAlex Tomas int err = 0; 820c9de560dSAlex Tomas int i; 821813e5727STheodore Ts'o ext4_group_t first_group, group; 822c9de560dSAlex Tomas int first_block; 823c9de560dSAlex Tomas struct super_block *sb; 824c9de560dSAlex Tomas struct buffer_head *bhs; 825fa77dcfaSDarrick J. Wong struct buffer_head **bh = NULL; 826c9de560dSAlex Tomas struct inode *inode; 827c9de560dSAlex Tomas char *data; 828c9de560dSAlex Tomas char *bitmap; 8299b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 830c9de560dSAlex Tomas 8316ba495e9STheodore Ts'o mb_debug(1, "init page %lu\n", page->index); 832c9de560dSAlex Tomas 833c9de560dSAlex Tomas inode = page->mapping->host; 834c9de560dSAlex Tomas sb = inode->i_sb; 8358df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 836c9de560dSAlex Tomas blocksize = 1 << inode->i_blkbits; 837c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / blocksize; 838c9de560dSAlex Tomas 839c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 840c9de560dSAlex Tomas if (groups_per_page == 0) 841c9de560dSAlex Tomas groups_per_page = 1; 842c9de560dSAlex Tomas 843c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 844c9de560dSAlex Tomas if (groups_per_page > 1) { 845c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 846c9de560dSAlex Tomas bh = kzalloc(i, GFP_NOFS); 847813e5727STheodore Ts'o if (bh == NULL) { 848813e5727STheodore Ts'o err = -ENOMEM; 849c9de560dSAlex Tomas goto out; 850813e5727STheodore Ts'o } 851c9de560dSAlex Tomas } else 852c9de560dSAlex Tomas bh = &bhs; 853c9de560dSAlex Tomas 854c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 855c9de560dSAlex Tomas 856c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 857813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 858813e5727STheodore Ts'o if (group >= ngroups) 859c9de560dSAlex Tomas break; 860c9de560dSAlex Tomas 861813e5727STheodore Ts'o grinfo = ext4_get_group_info(sb, group); 8629b8b7d35SAmir Goldstein /* 8639b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 8649b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 8659b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 8669b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 8679b8b7d35SAmir Goldstein */ 8689b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 8699b8b7d35SAmir Goldstein bh[i] = NULL; 8709b8b7d35SAmir Goldstein continue; 8719b8b7d35SAmir Goldstein } 872813e5727STheodore Ts'o if (!(bh[i] = ext4_read_block_bitmap_nowait(sb, group))) { 873c9de560dSAlex Tomas err = -ENOMEM; 874c9de560dSAlex Tomas goto out; 8752ccb5fb9SAneesh Kumar K.V } 876813e5727STheodore Ts'o mb_debug(1, "read bitmap for group %u\n", group); 877c9de560dSAlex Tomas } 878c9de560dSAlex Tomas 879c9de560dSAlex Tomas /* wait for I/O completion */ 880813e5727STheodore Ts'o for (i = 0, group = first_group; i < groups_per_page; i++, group++) { 881813e5727STheodore Ts'o if (bh[i] && ext4_wait_block_bitmap(sb, group, bh[i])) { 882c9de560dSAlex Tomas err = -EIO; 883c9de560dSAlex Tomas goto out; 884813e5727STheodore Ts'o } 885813e5727STheodore Ts'o } 886c9de560dSAlex Tomas 887c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 888c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 889c9de560dSAlex Tomas group = (first_block + i) >> 1; 8908df9675fSTheodore Ts'o if (group >= ngroups) 891c9de560dSAlex Tomas break; 892c9de560dSAlex Tomas 8939b8b7d35SAmir Goldstein if (!bh[group - first_group]) 8949b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 8959b8b7d35SAmir Goldstein continue; 8969b8b7d35SAmir Goldstein 897c9de560dSAlex Tomas /* 898c9de560dSAlex Tomas * data carry information regarding this 899c9de560dSAlex Tomas * particular group in the format specified 900c9de560dSAlex Tomas * above 901c9de560dSAlex Tomas * 902c9de560dSAlex Tomas */ 903c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 904c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 905c9de560dSAlex Tomas 906c9de560dSAlex Tomas /* 907c9de560dSAlex Tomas * We place the buddy block and bitmap block 908c9de560dSAlex Tomas * close together 909c9de560dSAlex Tomas */ 910c9de560dSAlex Tomas if ((first_block + i) & 1) { 911c9de560dSAlex Tomas /* this is block of buddy */ 912c9de560dSAlex Tomas BUG_ON(incore == NULL); 9136ba495e9STheodore Ts'o mb_debug(1, "put buddy for group %u in page %lu/%x\n", 914c9de560dSAlex Tomas group, page->index, i * blocksize); 915f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 916c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 917c9de560dSAlex Tomas grinfo->bb_fragments = 0; 918c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 9191927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 9201927805eSEric Sandeen (sb->s_blocksize_bits+2)); 921c9de560dSAlex Tomas /* 922c9de560dSAlex Tomas * incore got set to the group block bitmap below 923c9de560dSAlex Tomas */ 9247a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 9259b8b7d35SAmir Goldstein /* init the buddy */ 9269b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 927c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 9287a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 929c9de560dSAlex Tomas incore = NULL; 930c9de560dSAlex Tomas } else { 931c9de560dSAlex Tomas /* this is block of bitmap */ 932c9de560dSAlex Tomas BUG_ON(incore != NULL); 9336ba495e9STheodore Ts'o mb_debug(1, "put bitmap for group %u in page %lu/%x\n", 934c9de560dSAlex Tomas group, page->index, i * blocksize); 935f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 936c9de560dSAlex Tomas 937c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 938c9de560dSAlex Tomas ext4_lock_group(sb, group); 939c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 940c9de560dSAlex Tomas 941c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 942c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 9437a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 944c9de560dSAlex Tomas ext4_unlock_group(sb, group); 945c9de560dSAlex Tomas 946c9de560dSAlex Tomas /* set incore so that the buddy information can be 947c9de560dSAlex Tomas * generated using this 948c9de560dSAlex Tomas */ 949c9de560dSAlex Tomas incore = data; 950c9de560dSAlex Tomas } 951c9de560dSAlex Tomas } 952c9de560dSAlex Tomas SetPageUptodate(page); 953c9de560dSAlex Tomas 954c9de560dSAlex Tomas out: 955c9de560dSAlex Tomas if (bh) { 9569b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 957c9de560dSAlex Tomas brelse(bh[i]); 958c9de560dSAlex Tomas if (bh != &bhs) 959c9de560dSAlex Tomas kfree(bh); 960c9de560dSAlex Tomas } 961c9de560dSAlex Tomas return err; 962c9de560dSAlex Tomas } 963c9de560dSAlex Tomas 9648a57d9d6SCurt Wohlgemuth /* 9652de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 9662de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 9672de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 9682de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 969eee4adc7SEric Sandeen */ 9702de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 9712de8807bSAmir Goldstein ext4_group_t group, struct ext4_buddy *e4b) 972eee4adc7SEric Sandeen { 9732de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 9742de8807bSAmir Goldstein int block, pnum, poff; 975eee4adc7SEric Sandeen int blocks_per_page; 9762de8807bSAmir Goldstein struct page *page; 9772de8807bSAmir Goldstein 9782de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 9792de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 980eee4adc7SEric Sandeen 981eee4adc7SEric Sandeen blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 982eee4adc7SEric Sandeen /* 983eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 984eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 985eee4adc7SEric Sandeen * So for each group we need two blocks. 986eee4adc7SEric Sandeen */ 987eee4adc7SEric Sandeen block = group * 2; 988eee4adc7SEric Sandeen pnum = block / blocks_per_page; 9892de8807bSAmir Goldstein poff = block % blocks_per_page; 9902de8807bSAmir Goldstein page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 9912de8807bSAmir Goldstein if (!page) 9922de8807bSAmir Goldstein return -EIO; 9932de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 9942de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 9952de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 996eee4adc7SEric Sandeen 9972de8807bSAmir Goldstein if (blocks_per_page >= 2) { 9982de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 9992de8807bSAmir Goldstein return 0; 1000eee4adc7SEric Sandeen } 1001eee4adc7SEric Sandeen 10022de8807bSAmir Goldstein block++; 1003eee4adc7SEric Sandeen pnum = block / blocks_per_page; 10042de8807bSAmir Goldstein page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 10052de8807bSAmir Goldstein if (!page) 10062de8807bSAmir Goldstein return -EIO; 10072de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10082de8807bSAmir Goldstein e4b->bd_buddy_page = page; 10092de8807bSAmir Goldstein return 0; 1010eee4adc7SEric Sandeen } 1011eee4adc7SEric Sandeen 10122de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 10132de8807bSAmir Goldstein { 10142de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 10152de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 10162de8807bSAmir Goldstein page_cache_release(e4b->bd_bitmap_page); 10172de8807bSAmir Goldstein } 10182de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 10192de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 10202de8807bSAmir Goldstein page_cache_release(e4b->bd_buddy_page); 10212de8807bSAmir Goldstein } 1022eee4adc7SEric Sandeen } 1023eee4adc7SEric Sandeen 1024eee4adc7SEric Sandeen /* 10258a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10268a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10278a57d9d6SCurt Wohlgemuth * calling this routine! 10288a57d9d6SCurt Wohlgemuth */ 1029b6a758ecSAneesh Kumar K.V static noinline_for_stack 1030b6a758ecSAneesh Kumar K.V int ext4_mb_init_group(struct super_block *sb, ext4_group_t group) 1031b6a758ecSAneesh Kumar K.V { 1032b6a758ecSAneesh Kumar K.V 1033b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 10342de8807bSAmir Goldstein struct ext4_buddy e4b; 10352de8807bSAmir Goldstein struct page *page; 10362de8807bSAmir Goldstein int ret = 0; 1037b6a758ecSAneesh Kumar K.V 1038b10a44c3STheodore Ts'o might_sleep(); 1039b6a758ecSAneesh Kumar K.V mb_debug(1, "init group %u\n", group); 1040b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 1041b6a758ecSAneesh Kumar K.V /* 104208c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 104308c3a813SAneesh Kumar K.V * page which map to the group from which we are already 104408c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 104508c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 10462de8807bSAmir Goldstein * would have pinned buddy page to page cache. 1047b6a758ecSAneesh Kumar K.V */ 10482de8807bSAmir Goldstein ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b); 10492de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1050b6a758ecSAneesh Kumar K.V /* 1051b6a758ecSAneesh Kumar K.V * somebody initialized the group 1052b6a758ecSAneesh Kumar K.V * return without doing anything 1053b6a758ecSAneesh Kumar K.V */ 1054b6a758ecSAneesh Kumar K.V goto err; 1055b6a758ecSAneesh Kumar K.V } 10562de8807bSAmir Goldstein 10572de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1058b6a758ecSAneesh Kumar K.V ret = ext4_mb_init_cache(page, NULL); 10592de8807bSAmir Goldstein if (ret) 1060b6a758ecSAneesh Kumar K.V goto err; 10612de8807bSAmir Goldstein if (!PageUptodate(page)) { 1062b6a758ecSAneesh Kumar K.V ret = -EIO; 1063b6a758ecSAneesh Kumar K.V goto err; 1064b6a758ecSAneesh Kumar K.V } 1065b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 1066b6a758ecSAneesh Kumar K.V 10672de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1068b6a758ecSAneesh Kumar K.V /* 1069b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1070b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1071b6a758ecSAneesh Kumar K.V * init the buddy 1072b6a758ecSAneesh Kumar K.V */ 10732de8807bSAmir Goldstein ret = 0; 1074b6a758ecSAneesh Kumar K.V goto err; 1075b6a758ecSAneesh Kumar K.V } 10762de8807bSAmir Goldstein /* init buddy cache */ 10772de8807bSAmir Goldstein page = e4b.bd_buddy_page; 10782de8807bSAmir Goldstein ret = ext4_mb_init_cache(page, e4b.bd_bitmap); 10792de8807bSAmir Goldstein if (ret) 10802de8807bSAmir Goldstein goto err; 10812de8807bSAmir Goldstein if (!PageUptodate(page)) { 1082b6a758ecSAneesh Kumar K.V ret = -EIO; 1083b6a758ecSAneesh Kumar K.V goto err; 1084b6a758ecSAneesh Kumar K.V } 1085b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 1086b6a758ecSAneesh Kumar K.V err: 10872de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1088b6a758ecSAneesh Kumar K.V return ret; 1089b6a758ecSAneesh Kumar K.V } 1090b6a758ecSAneesh Kumar K.V 10918a57d9d6SCurt Wohlgemuth /* 10928a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10938a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10948a57d9d6SCurt Wohlgemuth * calling this routine! 10958a57d9d6SCurt Wohlgemuth */ 10964ddfef7bSEric Sandeen static noinline_for_stack int 10974ddfef7bSEric Sandeen ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1098c9de560dSAlex Tomas struct ext4_buddy *e4b) 1099c9de560dSAlex Tomas { 1100c9de560dSAlex Tomas int blocks_per_page; 1101c9de560dSAlex Tomas int block; 1102c9de560dSAlex Tomas int pnum; 1103c9de560dSAlex Tomas int poff; 1104c9de560dSAlex Tomas struct page *page; 1105fdf6c7a7SShen Feng int ret; 1106920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1107920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1108920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1109c9de560dSAlex Tomas 1110b10a44c3STheodore Ts'o might_sleep(); 11116ba495e9STheodore Ts'o mb_debug(1, "load group %u\n", group); 1112c9de560dSAlex Tomas 1113c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1114920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1115c9de560dSAlex Tomas 1116c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1117529da704STao Ma e4b->bd_info = grp; 1118c9de560dSAlex Tomas e4b->bd_sb = sb; 1119c9de560dSAlex Tomas e4b->bd_group = group; 1120c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1121c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1122c9de560dSAlex Tomas 1123f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1124f41c0750SAneesh Kumar K.V /* 1125f41c0750SAneesh Kumar K.V * we need full data about the group 1126f41c0750SAneesh Kumar K.V * to make a good selection 1127f41c0750SAneesh Kumar K.V */ 1128f41c0750SAneesh Kumar K.V ret = ext4_mb_init_group(sb, group); 1129f41c0750SAneesh Kumar K.V if (ret) 1130f41c0750SAneesh Kumar K.V return ret; 1131f41c0750SAneesh Kumar K.V } 1132f41c0750SAneesh Kumar K.V 1133c9de560dSAlex Tomas /* 1134c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1135c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1136c9de560dSAlex Tomas * So for each group we need two blocks. 1137c9de560dSAlex Tomas */ 1138c9de560dSAlex Tomas block = group * 2; 1139c9de560dSAlex Tomas pnum = block / blocks_per_page; 1140c9de560dSAlex Tomas poff = block % blocks_per_page; 1141c9de560dSAlex Tomas 1142c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1143c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 1144c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1145c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1146c9de560dSAlex Tomas if (page) 1147920313a7SAneesh Kumar K.V /* 1148920313a7SAneesh Kumar K.V * drop the page reference and try 1149920313a7SAneesh Kumar K.V * to get the page with lock. If we 1150920313a7SAneesh Kumar K.V * are not uptodate that implies 1151920313a7SAneesh Kumar K.V * somebody just created the page but 1152920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1153920313a7SAneesh Kumar K.V * wait for it to initialize. 1154920313a7SAneesh Kumar K.V */ 1155c9de560dSAlex Tomas page_cache_release(page); 1156c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1157c9de560dSAlex Tomas if (page) { 1158c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1159c9de560dSAlex Tomas if (!PageUptodate(page)) { 1160fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, NULL); 1161fdf6c7a7SShen Feng if (ret) { 1162fdf6c7a7SShen Feng unlock_page(page); 1163fdf6c7a7SShen Feng goto err; 1164fdf6c7a7SShen Feng } 1165c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1166c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1167c9de560dSAlex Tomas } 1168c9de560dSAlex Tomas unlock_page(page); 1169c9de560dSAlex Tomas } 1170c9de560dSAlex Tomas } 1171fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1172fdf6c7a7SShen Feng ret = -EIO; 1173c9de560dSAlex Tomas goto err; 1174fdf6c7a7SShen Feng } 1175c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1176c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1177c9de560dSAlex Tomas mark_page_accessed(page); 1178c9de560dSAlex Tomas 1179c9de560dSAlex Tomas block++; 1180c9de560dSAlex Tomas pnum = block / blocks_per_page; 1181c9de560dSAlex Tomas poff = block % blocks_per_page; 1182c9de560dSAlex Tomas 1183c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1184c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1185c9de560dSAlex Tomas if (page) 1186c9de560dSAlex Tomas page_cache_release(page); 1187c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1188c9de560dSAlex Tomas if (page) { 1189c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1190fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1191fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, e4b->bd_bitmap); 1192fdf6c7a7SShen Feng if (ret) { 1193fdf6c7a7SShen Feng unlock_page(page); 1194fdf6c7a7SShen Feng goto err; 1195fdf6c7a7SShen Feng } 1196fdf6c7a7SShen Feng } 1197c9de560dSAlex Tomas unlock_page(page); 1198c9de560dSAlex Tomas } 1199c9de560dSAlex Tomas } 1200fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1201fdf6c7a7SShen Feng ret = -EIO; 1202c9de560dSAlex Tomas goto err; 1203fdf6c7a7SShen Feng } 1204c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1205c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1206c9de560dSAlex Tomas mark_page_accessed(page); 1207c9de560dSAlex Tomas 1208c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 1209c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 1210c9de560dSAlex Tomas 1211c9de560dSAlex Tomas return 0; 1212c9de560dSAlex Tomas 1213c9de560dSAlex Tomas err: 121426626f11SYang Ruirui if (page) 121526626f11SYang Ruirui page_cache_release(page); 1216c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1217c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1218c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1219c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1220c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1221c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1222fdf6c7a7SShen Feng return ret; 1223c9de560dSAlex Tomas } 1224c9de560dSAlex Tomas 1225e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1226c9de560dSAlex Tomas { 1227c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1228c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1229c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1230c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1231c9de560dSAlex Tomas } 1232c9de560dSAlex Tomas 1233c9de560dSAlex Tomas 1234c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1235c9de560dSAlex Tomas { 1236c9de560dSAlex Tomas int order = 1; 1237c9de560dSAlex Tomas void *bb; 1238c9de560dSAlex Tomas 1239c5e8f3f3STheodore Ts'o BUG_ON(e4b->bd_bitmap == e4b->bd_buddy); 1240c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1241c9de560dSAlex Tomas 1242c5e8f3f3STheodore Ts'o bb = e4b->bd_buddy; 1243c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1244c9de560dSAlex Tomas block = block >> 1; 1245c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1246c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1247c9de560dSAlex Tomas return order; 1248c9de560dSAlex Tomas } 1249c9de560dSAlex Tomas bb += 1 << (e4b->bd_blkbits - order); 1250c9de560dSAlex Tomas order++; 1251c9de560dSAlex Tomas } 1252c9de560dSAlex Tomas return 0; 1253c9de560dSAlex Tomas } 1254c9de560dSAlex Tomas 1255955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1256c9de560dSAlex Tomas { 1257c9de560dSAlex Tomas __u32 *addr; 1258c9de560dSAlex Tomas 1259c9de560dSAlex Tomas len = cur + len; 1260c9de560dSAlex Tomas while (cur < len) { 1261c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1262c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1263c9de560dSAlex Tomas addr = bm + (cur >> 3); 1264c9de560dSAlex Tomas *addr = 0; 1265c9de560dSAlex Tomas cur += 32; 1266c9de560dSAlex Tomas continue; 1267c9de560dSAlex Tomas } 1268e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1269c9de560dSAlex Tomas cur++; 1270c9de560dSAlex Tomas } 1271c9de560dSAlex Tomas } 1272c9de560dSAlex Tomas 1273eabe0444SAndrey Sidorov /* clear bits in given range 1274eabe0444SAndrey Sidorov * will return first found zero bit if any, -1 otherwise 1275eabe0444SAndrey Sidorov */ 1276eabe0444SAndrey Sidorov static int mb_test_and_clear_bits(void *bm, int cur, int len) 1277eabe0444SAndrey Sidorov { 1278eabe0444SAndrey Sidorov __u32 *addr; 1279eabe0444SAndrey Sidorov int zero_bit = -1; 1280eabe0444SAndrey Sidorov 1281eabe0444SAndrey Sidorov len = cur + len; 1282eabe0444SAndrey Sidorov while (cur < len) { 1283eabe0444SAndrey Sidorov if ((cur & 31) == 0 && (len - cur) >= 32) { 1284eabe0444SAndrey Sidorov /* fast path: clear whole word at once */ 1285eabe0444SAndrey Sidorov addr = bm + (cur >> 3); 1286eabe0444SAndrey Sidorov if (*addr != (__u32)(-1) && zero_bit == -1) 1287eabe0444SAndrey Sidorov zero_bit = cur + mb_find_next_zero_bit(addr, 32, 0); 1288eabe0444SAndrey Sidorov *addr = 0; 1289eabe0444SAndrey Sidorov cur += 32; 1290eabe0444SAndrey Sidorov continue; 1291eabe0444SAndrey Sidorov } 1292eabe0444SAndrey Sidorov if (!mb_test_and_clear_bit(cur, bm) && zero_bit == -1) 1293eabe0444SAndrey Sidorov zero_bit = cur; 1294eabe0444SAndrey Sidorov cur++; 1295eabe0444SAndrey Sidorov } 1296eabe0444SAndrey Sidorov 1297eabe0444SAndrey Sidorov return zero_bit; 1298eabe0444SAndrey Sidorov } 1299eabe0444SAndrey Sidorov 1300c3e94d1dSYongqiang Yang void ext4_set_bits(void *bm, int cur, int len) 1301c9de560dSAlex Tomas { 1302c9de560dSAlex Tomas __u32 *addr; 1303c9de560dSAlex Tomas 1304c9de560dSAlex Tomas len = cur + len; 1305c9de560dSAlex Tomas while (cur < len) { 1306c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1307c9de560dSAlex Tomas /* fast path: set whole word at once */ 1308c9de560dSAlex Tomas addr = bm + (cur >> 3); 1309c9de560dSAlex Tomas *addr = 0xffffffff; 1310c9de560dSAlex Tomas cur += 32; 1311c9de560dSAlex Tomas continue; 1312c9de560dSAlex Tomas } 1313e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1314c9de560dSAlex Tomas cur++; 1315c9de560dSAlex Tomas } 1316c9de560dSAlex Tomas } 1317c9de560dSAlex Tomas 1318eabe0444SAndrey Sidorov /* 1319eabe0444SAndrey Sidorov * _________________________________________________________________ */ 1320eabe0444SAndrey Sidorov 1321eabe0444SAndrey Sidorov static inline int mb_buddy_adjust_border(int* bit, void* bitmap, int side) 1322eabe0444SAndrey Sidorov { 1323eabe0444SAndrey Sidorov if (mb_test_bit(*bit + side, bitmap)) { 1324eabe0444SAndrey Sidorov mb_clear_bit(*bit, bitmap); 1325eabe0444SAndrey Sidorov (*bit) -= side; 1326eabe0444SAndrey Sidorov return 1; 1327eabe0444SAndrey Sidorov } 1328eabe0444SAndrey Sidorov else { 1329eabe0444SAndrey Sidorov (*bit) += side; 1330eabe0444SAndrey Sidorov mb_set_bit(*bit, bitmap); 1331eabe0444SAndrey Sidorov return -1; 1332eabe0444SAndrey Sidorov } 1333eabe0444SAndrey Sidorov } 1334eabe0444SAndrey Sidorov 1335eabe0444SAndrey Sidorov static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last) 1336eabe0444SAndrey Sidorov { 1337eabe0444SAndrey Sidorov int max; 1338eabe0444SAndrey Sidorov int order = 1; 1339eabe0444SAndrey Sidorov void *buddy = mb_find_buddy(e4b, order, &max); 1340eabe0444SAndrey Sidorov 1341eabe0444SAndrey Sidorov while (buddy) { 1342eabe0444SAndrey Sidorov void *buddy2; 1343eabe0444SAndrey Sidorov 1344eabe0444SAndrey Sidorov /* Bits in range [first; last] are known to be set since 1345eabe0444SAndrey Sidorov * corresponding blocks were allocated. Bits in range 1346eabe0444SAndrey Sidorov * (first; last) will stay set because they form buddies on 1347eabe0444SAndrey Sidorov * upper layer. We just deal with borders if they don't 1348eabe0444SAndrey Sidorov * align with upper layer and then go up. 1349eabe0444SAndrey Sidorov * Releasing entire group is all about clearing 1350eabe0444SAndrey Sidorov * single bit of highest order buddy. 1351eabe0444SAndrey Sidorov */ 1352eabe0444SAndrey Sidorov 1353eabe0444SAndrey Sidorov /* Example: 1354eabe0444SAndrey Sidorov * --------------------------------- 1355eabe0444SAndrey Sidorov * | 1 | 1 | 1 | 1 | 1356eabe0444SAndrey Sidorov * --------------------------------- 1357eabe0444SAndrey Sidorov * | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | 1358eabe0444SAndrey Sidorov * --------------------------------- 1359eabe0444SAndrey Sidorov * 0 1 2 3 4 5 6 7 1360eabe0444SAndrey Sidorov * \_____________________/ 1361eabe0444SAndrey Sidorov * 1362eabe0444SAndrey Sidorov * Neither [1] nor [6] is aligned to above layer. 1363eabe0444SAndrey Sidorov * Left neighbour [0] is free, so mark it busy, 1364eabe0444SAndrey Sidorov * decrease bb_counters and extend range to 1365eabe0444SAndrey Sidorov * [0; 6] 1366eabe0444SAndrey Sidorov * Right neighbour [7] is busy. It can't be coaleasced with [6], so 1367eabe0444SAndrey Sidorov * mark [6] free, increase bb_counters and shrink range to 1368eabe0444SAndrey Sidorov * [0; 5]. 1369eabe0444SAndrey Sidorov * Then shift range to [0; 2], go up and do the same. 1370eabe0444SAndrey Sidorov */ 1371eabe0444SAndrey Sidorov 1372eabe0444SAndrey Sidorov 1373eabe0444SAndrey Sidorov if (first & 1) 1374eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1); 1375eabe0444SAndrey Sidorov if (!(last & 1)) 1376eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1); 1377eabe0444SAndrey Sidorov if (first > last) 1378eabe0444SAndrey Sidorov break; 1379eabe0444SAndrey Sidorov order++; 1380eabe0444SAndrey Sidorov 1381eabe0444SAndrey Sidorov if (first == last || !(buddy2 = mb_find_buddy(e4b, order, &max))) { 1382eabe0444SAndrey Sidorov mb_clear_bits(buddy, first, last - first + 1); 1383eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[order - 1] += last - first + 1; 1384eabe0444SAndrey Sidorov break; 1385eabe0444SAndrey Sidorov } 1386eabe0444SAndrey Sidorov first >>= 1; 1387eabe0444SAndrey Sidorov last >>= 1; 1388eabe0444SAndrey Sidorov buddy = buddy2; 1389eabe0444SAndrey Sidorov } 1390eabe0444SAndrey Sidorov } 1391eabe0444SAndrey Sidorov 13927e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1393c9de560dSAlex Tomas int first, int count) 1394c9de560dSAlex Tomas { 1395eabe0444SAndrey Sidorov int left_is_free = 0; 1396eabe0444SAndrey Sidorov int right_is_free = 0; 1397eabe0444SAndrey Sidorov int block; 1398eabe0444SAndrey Sidorov int last = first + count - 1; 1399c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1400c9de560dSAlex Tomas 1401eabe0444SAndrey Sidorov BUG_ON(last >= (sb->s_blocksize << 3)); 1402bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1403163a203dSDarrick J. Wong /* Don't bother if the block group is corrupt. */ 1404163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) 1405163a203dSDarrick J. Wong return; 1406163a203dSDarrick J. Wong 1407c9de560dSAlex Tomas mb_check_buddy(e4b); 1408c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1409c9de560dSAlex Tomas 1410c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1411c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1412c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1413c9de560dSAlex Tomas 1414eabe0444SAndrey Sidorov /* access memory sequentially: check left neighbour, 1415eabe0444SAndrey Sidorov * clear range and then check right neighbour 1416eabe0444SAndrey Sidorov */ 1417c9de560dSAlex Tomas if (first != 0) 1418eabe0444SAndrey Sidorov left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap); 1419eabe0444SAndrey Sidorov block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count); 1420eabe0444SAndrey Sidorov if (last + 1 < EXT4_SB(sb)->s_mb_maxs[0]) 1421eabe0444SAndrey Sidorov right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap); 1422c9de560dSAlex Tomas 1423eabe0444SAndrey Sidorov if (unlikely(block != -1)) { 1424c9de560dSAlex Tomas ext4_fsblk_t blocknr; 14255661bd68SAkinobu Mita 14265661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 142753accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), block); 14285d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1429e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1430e29136f8STheodore Ts'o blocknr, 1431e29136f8STheodore Ts'o "freeing already freed block " 1432163a203dSDarrick J. Wong "(bit %u); block bitmap corrupt.", 1433163a203dSDarrick J. Wong block); 1434163a203dSDarrick J. Wong /* Mark the block group as corrupt. */ 1435163a203dSDarrick J. Wong set_bit(EXT4_GROUP_INFO_BBITMAP_CORRUPT_BIT, 1436163a203dSDarrick J. Wong &e4b->bd_info->bb_state); 1437eabe0444SAndrey Sidorov mb_regenerate_buddy(e4b); 1438eabe0444SAndrey Sidorov goto done; 1439c9de560dSAlex Tomas } 1440c9de560dSAlex Tomas 1441eabe0444SAndrey Sidorov /* let's maintain fragments counter */ 1442eabe0444SAndrey Sidorov if (left_is_free && right_is_free) 1443eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments--; 1444eabe0444SAndrey Sidorov else if (!left_is_free && !right_is_free) 1445eabe0444SAndrey Sidorov e4b->bd_info->bb_fragments++; 1446c9de560dSAlex Tomas 1447eabe0444SAndrey Sidorov /* buddy[0] == bd_bitmap is a special case, so handle 1448eabe0444SAndrey Sidorov * it right away and let mb_buddy_mark_free stay free of 1449eabe0444SAndrey Sidorov * zero order checks. 1450eabe0444SAndrey Sidorov * Check if neighbours are to be coaleasced, 1451eabe0444SAndrey Sidorov * adjust bitmap bb_counters and borders appropriately. 1452eabe0444SAndrey Sidorov */ 1453eabe0444SAndrey Sidorov if (first & 1) { 1454eabe0444SAndrey Sidorov first += !left_is_free; 1455eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1; 1456c9de560dSAlex Tomas } 1457eabe0444SAndrey Sidorov if (!(last & 1)) { 1458eabe0444SAndrey Sidorov last -= !right_is_free; 1459eabe0444SAndrey Sidorov e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1; 1460c9de560dSAlex Tomas } 1461eabe0444SAndrey Sidorov 1462eabe0444SAndrey Sidorov if (first <= last) 1463eabe0444SAndrey Sidorov mb_buddy_mark_free(e4b, first >> 1, last >> 1); 1464eabe0444SAndrey Sidorov 1465eabe0444SAndrey Sidorov done: 14668a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1467c9de560dSAlex Tomas mb_check_buddy(e4b); 1468c9de560dSAlex Tomas } 1469c9de560dSAlex Tomas 147015c006a2SRobin Dong static int mb_find_extent(struct ext4_buddy *e4b, int block, 1471c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1472c9de560dSAlex Tomas { 1473c9de560dSAlex Tomas int next = block; 147415c006a2SRobin Dong int max, order; 1475c9de560dSAlex Tomas void *buddy; 1476c9de560dSAlex Tomas 1477bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1478c9de560dSAlex Tomas BUG_ON(ex == NULL); 1479c9de560dSAlex Tomas 148015c006a2SRobin Dong buddy = mb_find_buddy(e4b, 0, &max); 1481c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1482c9de560dSAlex Tomas BUG_ON(block >= max); 1483c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1484c9de560dSAlex Tomas ex->fe_len = 0; 1485c9de560dSAlex Tomas ex->fe_start = 0; 1486c9de560dSAlex Tomas ex->fe_group = 0; 1487c9de560dSAlex Tomas return 0; 1488c9de560dSAlex Tomas } 1489c9de560dSAlex Tomas 1490c9de560dSAlex Tomas /* find actual order */ 1491c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1492c9de560dSAlex Tomas block = block >> order; 1493c9de560dSAlex Tomas 1494c9de560dSAlex Tomas ex->fe_len = 1 << order; 1495c9de560dSAlex Tomas ex->fe_start = block << order; 1496c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1497c9de560dSAlex Tomas 1498c9de560dSAlex Tomas /* calc difference from given start */ 1499c9de560dSAlex Tomas next = next - ex->fe_start; 1500c9de560dSAlex Tomas ex->fe_len -= next; 1501c9de560dSAlex Tomas ex->fe_start += next; 1502c9de560dSAlex Tomas 1503c9de560dSAlex Tomas while (needed > ex->fe_len && 1504d8ec0c39SAlan Cox mb_find_buddy(e4b, order, &max)) { 1505c9de560dSAlex Tomas 1506c9de560dSAlex Tomas if (block + 1 >= max) 1507c9de560dSAlex Tomas break; 1508c9de560dSAlex Tomas 1509c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1510c5e8f3f3STheodore Ts'o if (mb_test_bit(next, e4b->bd_bitmap)) 1511c9de560dSAlex Tomas break; 1512c9de560dSAlex Tomas 1513b051d8dcSRobin Dong order = mb_find_order_for_block(e4b, next); 1514c9de560dSAlex Tomas 1515c9de560dSAlex Tomas block = next >> order; 1516c9de560dSAlex Tomas ex->fe_len += 1 << order; 1517c9de560dSAlex Tomas } 1518c9de560dSAlex Tomas 1519c9de560dSAlex Tomas BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3))); 1520c9de560dSAlex Tomas return ex->fe_len; 1521c9de560dSAlex Tomas } 1522c9de560dSAlex Tomas 1523c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1524c9de560dSAlex Tomas { 1525c9de560dSAlex Tomas int ord; 1526c9de560dSAlex Tomas int mlen = 0; 1527c9de560dSAlex Tomas int max = 0; 1528c9de560dSAlex Tomas int cur; 1529c9de560dSAlex Tomas int start = ex->fe_start; 1530c9de560dSAlex Tomas int len = ex->fe_len; 1531c9de560dSAlex Tomas unsigned ret = 0; 1532c9de560dSAlex Tomas int len0 = len; 1533c9de560dSAlex Tomas void *buddy; 1534c9de560dSAlex Tomas 1535c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1536c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1537bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1538c9de560dSAlex Tomas mb_check_buddy(e4b); 1539c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1540c9de560dSAlex Tomas 1541c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1542c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1543c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1544c9de560dSAlex Tomas 1545c9de560dSAlex Tomas /* let's maintain fragments counter */ 1546c9de560dSAlex Tomas if (start != 0) 1547c5e8f3f3STheodore Ts'o mlen = !mb_test_bit(start - 1, e4b->bd_bitmap); 1548c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1549c5e8f3f3STheodore Ts'o max = !mb_test_bit(start + len, e4b->bd_bitmap); 1550c9de560dSAlex Tomas if (mlen && max) 1551c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1552c9de560dSAlex Tomas else if (!mlen && !max) 1553c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1554c9de560dSAlex Tomas 1555c9de560dSAlex Tomas /* let's maintain buddy itself */ 1556c9de560dSAlex Tomas while (len) { 1557c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1558c9de560dSAlex Tomas 1559c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1560c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1561c9de560dSAlex Tomas mlen = 1 << ord; 1562c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1563c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1564c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1565c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1566c9de560dSAlex Tomas start += mlen; 1567c9de560dSAlex Tomas len -= mlen; 1568c9de560dSAlex Tomas BUG_ON(len < 0); 1569c9de560dSAlex Tomas continue; 1570c9de560dSAlex Tomas } 1571c9de560dSAlex Tomas 1572c9de560dSAlex Tomas /* store for history */ 1573c9de560dSAlex Tomas if (ret == 0) 1574c9de560dSAlex Tomas ret = len | (ord << 16); 1575c9de560dSAlex Tomas 1576c9de560dSAlex Tomas /* we have to split large buddy */ 1577c9de560dSAlex Tomas BUG_ON(ord <= 0); 1578c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1579c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1580c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1581c9de560dSAlex Tomas 1582c9de560dSAlex Tomas ord--; 1583c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1584c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1585c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1586c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1587c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1588c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1589c9de560dSAlex Tomas } 15908a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 1591c9de560dSAlex Tomas 1592c5e8f3f3STheodore Ts'o ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0); 1593c9de560dSAlex Tomas mb_check_buddy(e4b); 1594c9de560dSAlex Tomas 1595c9de560dSAlex Tomas return ret; 1596c9de560dSAlex Tomas } 1597c9de560dSAlex Tomas 1598c9de560dSAlex Tomas /* 1599c9de560dSAlex Tomas * Must be called under group lock! 1600c9de560dSAlex Tomas */ 1601c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1602c9de560dSAlex Tomas struct ext4_buddy *e4b) 1603c9de560dSAlex Tomas { 1604c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1605c9de560dSAlex Tomas int ret; 1606c9de560dSAlex Tomas 1607c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1608c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1609c9de560dSAlex Tomas 1610c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1611c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1612c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1613c9de560dSAlex Tomas 1614c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1615c9de560dSAlex Tomas * allocated blocks for history */ 1616c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1617c9de560dSAlex Tomas 1618c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1619c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1620c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1621c9de560dSAlex Tomas 1622c3a326a6SAneesh Kumar K.V /* 1623c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 1624c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 1625c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 1626c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 1627c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 1628c3a326a6SAneesh Kumar K.V */ 1629c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1630c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1631c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1632c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 1633c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 16344ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 1635c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1636c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1637c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1638c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1639c9de560dSAlex Tomas } 1640c9de560dSAlex Tomas } 1641c9de560dSAlex Tomas 1642c9de560dSAlex Tomas /* 1643c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1644c9de560dSAlex Tomas */ 1645c9de560dSAlex Tomas 1646c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1647c9de560dSAlex Tomas struct ext4_buddy *e4b, 1648c9de560dSAlex Tomas int finish_group) 1649c9de560dSAlex Tomas { 1650c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1651c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1652c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1653c9de560dSAlex Tomas struct ext4_free_extent ex; 1654c9de560dSAlex Tomas int max; 1655c9de560dSAlex Tomas 1656032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 1657032115fcSAneesh Kumar K.V return; 1658c9de560dSAlex Tomas /* 1659c9de560dSAlex Tomas * We don't want to scan for a whole year 1660c9de560dSAlex Tomas */ 1661c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1662c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1663c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1664c9de560dSAlex Tomas return; 1665c9de560dSAlex Tomas } 1666c9de560dSAlex Tomas 1667c9de560dSAlex Tomas /* 1668c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1669c9de560dSAlex Tomas */ 1670c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1671c9de560dSAlex Tomas return; 1672c9de560dSAlex Tomas 1673c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1674c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1675c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1676c9de560dSAlex Tomas * when it was found (within this lock-unlock 1677c9de560dSAlex Tomas * period or not) */ 167815c006a2SRobin Dong max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex); 1679c9de560dSAlex Tomas if (max >= gex->fe_len) { 1680c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1681c9de560dSAlex Tomas return; 1682c9de560dSAlex Tomas } 1683c9de560dSAlex Tomas } 1684c9de560dSAlex Tomas } 1685c9de560dSAlex Tomas 1686c9de560dSAlex Tomas /* 1687c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1688c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1689c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1690c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1691c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1692c9de560dSAlex Tomas * mballoc can't find good enough extent. 1693c9de560dSAlex Tomas * 1694c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1695c9de560dSAlex Tomas */ 1696c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1697c9de560dSAlex Tomas struct ext4_free_extent *ex, 1698c9de560dSAlex Tomas struct ext4_buddy *e4b) 1699c9de560dSAlex Tomas { 1700c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1701c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1702c9de560dSAlex Tomas 1703c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 17047137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 17057137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 1706c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1707c9de560dSAlex Tomas 1708c9de560dSAlex Tomas ac->ac_found++; 1709c9de560dSAlex Tomas 1710c9de560dSAlex Tomas /* 1711c9de560dSAlex Tomas * The special case - take what you catch first 1712c9de560dSAlex Tomas */ 1713c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1714c9de560dSAlex Tomas *bex = *ex; 1715c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1716c9de560dSAlex Tomas return; 1717c9de560dSAlex Tomas } 1718c9de560dSAlex Tomas 1719c9de560dSAlex Tomas /* 1720c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1721c9de560dSAlex Tomas */ 1722c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1723c9de560dSAlex Tomas *bex = *ex; 1724c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1725c9de560dSAlex Tomas return; 1726c9de560dSAlex Tomas } 1727c9de560dSAlex Tomas 1728c9de560dSAlex Tomas /* 1729c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1730c9de560dSAlex Tomas */ 1731c9de560dSAlex Tomas if (bex->fe_len == 0) { 1732c9de560dSAlex Tomas *bex = *ex; 1733c9de560dSAlex Tomas return; 1734c9de560dSAlex Tomas } 1735c9de560dSAlex Tomas 1736c9de560dSAlex Tomas /* 1737c9de560dSAlex Tomas * If new found extent is better, store it in the context 1738c9de560dSAlex Tomas */ 1739c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1740c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1741c9de560dSAlex Tomas * larger than previous best one is better */ 1742c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1743c9de560dSAlex Tomas *bex = *ex; 1744c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1745c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1746c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1747c9de560dSAlex Tomas * smaller than previous one */ 1748c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1749c9de560dSAlex Tomas *bex = *ex; 1750c9de560dSAlex Tomas } 1751c9de560dSAlex Tomas 1752c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1753c9de560dSAlex Tomas } 1754c9de560dSAlex Tomas 1755089ceeccSEric Sandeen static noinline_for_stack 1756089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1757c9de560dSAlex Tomas struct ext4_buddy *e4b) 1758c9de560dSAlex Tomas { 1759c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1760c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1761c9de560dSAlex Tomas int max; 1762c9de560dSAlex Tomas int err; 1763c9de560dSAlex Tomas 1764c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1765c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1766c9de560dSAlex Tomas if (err) 1767c9de560dSAlex Tomas return err; 1768c9de560dSAlex Tomas 1769c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 177015c006a2SRobin Dong max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex); 1771c9de560dSAlex Tomas 1772c9de560dSAlex Tomas if (max > 0) { 1773c9de560dSAlex Tomas ac->ac_b_ex = ex; 1774c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1775c9de560dSAlex Tomas } 1776c9de560dSAlex Tomas 1777c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1778e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1779c9de560dSAlex Tomas 1780c9de560dSAlex Tomas return 0; 1781c9de560dSAlex Tomas } 1782c9de560dSAlex Tomas 1783089ceeccSEric Sandeen static noinline_for_stack 1784089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1785c9de560dSAlex Tomas struct ext4_buddy *e4b) 1786c9de560dSAlex Tomas { 1787c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1788c9de560dSAlex Tomas int max; 1789c9de560dSAlex Tomas int err; 1790c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1791838cd0cfSYongqiang Yang struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1792c9de560dSAlex Tomas struct ext4_free_extent ex; 1793c9de560dSAlex Tomas 1794c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1795c9de560dSAlex Tomas return 0; 1796838cd0cfSYongqiang Yang if (grp->bb_free == 0) 1797838cd0cfSYongqiang Yang return 0; 1798c9de560dSAlex Tomas 1799c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1800c9de560dSAlex Tomas if (err) 1801c9de560dSAlex Tomas return err; 1802c9de560dSAlex Tomas 1803163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info))) { 1804163a203dSDarrick J. Wong ext4_mb_unload_buddy(e4b); 1805163a203dSDarrick J. Wong return 0; 1806163a203dSDarrick J. Wong } 1807163a203dSDarrick J. Wong 1808c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 180915c006a2SRobin Dong max = mb_find_extent(e4b, ac->ac_g_ex.fe_start, 1810c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1811ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADFA11; /* debug value */ 1812c9de560dSAlex Tomas 1813c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1814c9de560dSAlex Tomas ext4_fsblk_t start; 1815c9de560dSAlex Tomas 18165661bd68SAkinobu Mita start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + 18175661bd68SAkinobu Mita ex.fe_start; 1818c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1819c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1820c9de560dSAlex Tomas ac->ac_found++; 1821c9de560dSAlex Tomas ac->ac_b_ex = ex; 1822c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1823c9de560dSAlex Tomas } 1824c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1825c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1826c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1827c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1828c9de560dSAlex Tomas ac->ac_found++; 1829c9de560dSAlex Tomas ac->ac_b_ex = ex; 1830c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1831c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1832c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1833c9de560dSAlex Tomas * number of blocks to an existing extent */ 1834c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1835c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1836c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1837c9de560dSAlex Tomas ac->ac_found++; 1838c9de560dSAlex Tomas ac->ac_b_ex = ex; 1839c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1840c9de560dSAlex Tomas } 1841c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1842e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1843c9de560dSAlex Tomas 1844c9de560dSAlex Tomas return 0; 1845c9de560dSAlex Tomas } 1846c9de560dSAlex Tomas 1847c9de560dSAlex Tomas /* 1848c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1849c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1850c9de560dSAlex Tomas */ 1851089ceeccSEric Sandeen static noinline_for_stack 1852089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1853c9de560dSAlex Tomas struct ext4_buddy *e4b) 1854c9de560dSAlex Tomas { 1855c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1856c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1857c9de560dSAlex Tomas void *buddy; 1858c9de560dSAlex Tomas int i; 1859c9de560dSAlex Tomas int k; 1860c9de560dSAlex Tomas int max; 1861c9de560dSAlex Tomas 1862c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1863c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1864c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1865c9de560dSAlex Tomas continue; 1866c9de560dSAlex Tomas 1867c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1868c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1869c9de560dSAlex Tomas 1870ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 1871c9de560dSAlex Tomas BUG_ON(k >= max); 1872c9de560dSAlex Tomas 1873c9de560dSAlex Tomas ac->ac_found++; 1874c9de560dSAlex Tomas 1875c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1876c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1877c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1878c9de560dSAlex Tomas 1879c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1880c9de560dSAlex Tomas 1881c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); 1882c9de560dSAlex Tomas 1883c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1884c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1885c9de560dSAlex Tomas 1886c9de560dSAlex Tomas break; 1887c9de560dSAlex Tomas } 1888c9de560dSAlex Tomas } 1889c9de560dSAlex Tomas 1890c9de560dSAlex Tomas /* 1891c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 1892c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 1893c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 1894c9de560dSAlex Tomas */ 1895089ceeccSEric Sandeen static noinline_for_stack 1896089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 1897c9de560dSAlex Tomas struct ext4_buddy *e4b) 1898c9de560dSAlex Tomas { 1899c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1900c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 1901c9de560dSAlex Tomas struct ext4_free_extent ex; 1902c9de560dSAlex Tomas int i; 1903c9de560dSAlex Tomas int free; 1904c9de560dSAlex Tomas 1905c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 1906c9de560dSAlex Tomas BUG_ON(free <= 0); 1907c9de560dSAlex Tomas 1908c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 1909c9de560dSAlex Tomas 1910c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 1911ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 19127137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 19137137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 191426346ff6SAneesh Kumar K.V /* 1915e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 191626346ff6SAneesh Kumar K.V * free blocks even though group info says we 191726346ff6SAneesh Kumar K.V * we have free blocks 191826346ff6SAneesh Kumar K.V */ 1919e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 192053accfa9STheodore Ts'o "%d free clusters as per " 1921fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 192226346ff6SAneesh Kumar K.V free); 1923c9de560dSAlex Tomas break; 1924c9de560dSAlex Tomas } 1925c9de560dSAlex Tomas 192615c006a2SRobin Dong mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex); 1927c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 192826346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 1929e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 193053accfa9STheodore Ts'o "%d free clusters as per " 1931fde4d95aSTheodore Ts'o "group info. But got %d blocks", 193226346ff6SAneesh Kumar K.V free, ex.fe_len); 1933e56eb659SAneesh Kumar K.V /* 1934e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 1935e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 1936e56eb659SAneesh Kumar K.V * without claiming the space. 1937e56eb659SAneesh Kumar K.V */ 1938e56eb659SAneesh Kumar K.V break; 193926346ff6SAneesh Kumar K.V } 1940ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADC0DE; /* debug value */ 1941c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 1942c9de560dSAlex Tomas 1943c9de560dSAlex Tomas i += ex.fe_len; 1944c9de560dSAlex Tomas free -= ex.fe_len; 1945c9de560dSAlex Tomas } 1946c9de560dSAlex Tomas 1947c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 1948c9de560dSAlex Tomas } 1949c9de560dSAlex Tomas 1950c9de560dSAlex Tomas /* 1951c9de560dSAlex Tomas * This is a special case for storages like raid5 1952506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 1953c9de560dSAlex Tomas */ 1954089ceeccSEric Sandeen static noinline_for_stack 1955089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 1956c9de560dSAlex Tomas struct ext4_buddy *e4b) 1957c9de560dSAlex Tomas { 1958c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1959c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 1960c5e8f3f3STheodore Ts'o void *bitmap = e4b->bd_bitmap; 1961c9de560dSAlex Tomas struct ext4_free_extent ex; 1962c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 1963c9de560dSAlex Tomas ext4_fsblk_t a; 1964c9de560dSAlex Tomas ext4_grpblk_t i; 1965c9de560dSAlex Tomas int max; 1966c9de560dSAlex Tomas 1967c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 1968c9de560dSAlex Tomas 1969c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 19705661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 19715661bd68SAkinobu Mita 1972c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 1973c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 1974c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 1975c9de560dSAlex Tomas 19767137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 1977c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 197815c006a2SRobin Dong max = mb_find_extent(e4b, i, sbi->s_stripe, &ex); 1979c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 1980c9de560dSAlex Tomas ac->ac_found++; 1981ab0c00fcSTheodore Ts'o ex.fe_logical = 0xDEADF00D; /* debug value */ 1982c9de560dSAlex Tomas ac->ac_b_ex = ex; 1983c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1984c9de560dSAlex Tomas break; 1985c9de560dSAlex Tomas } 1986c9de560dSAlex Tomas } 1987c9de560dSAlex Tomas i += sbi->s_stripe; 1988c9de560dSAlex Tomas } 1989c9de560dSAlex Tomas } 1990c9de560dSAlex Tomas 19918a57d9d6SCurt Wohlgemuth /* This is now called BEFORE we load the buddy bitmap. */ 1992c9de560dSAlex Tomas static int ext4_mb_good_group(struct ext4_allocation_context *ac, 1993c9de560dSAlex Tomas ext4_group_t group, int cr) 1994c9de560dSAlex Tomas { 1995c9de560dSAlex Tomas unsigned free, fragments; 1996a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 1997c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1998c9de560dSAlex Tomas 1999c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 20008a57d9d6SCurt Wohlgemuth 200101fc48e8STheodore Ts'o free = grp->bb_free; 200201fc48e8STheodore Ts'o if (free == 0) 200301fc48e8STheodore Ts'o return 0; 200401fc48e8STheodore Ts'o if (cr <= 2 && free < ac->ac_g_ex.fe_len) 200501fc48e8STheodore Ts'o return 0; 200601fc48e8STheodore Ts'o 2007163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(grp))) 2008163a203dSDarrick J. Wong return 0; 2009163a203dSDarrick J. Wong 20108a57d9d6SCurt Wohlgemuth /* We only do this if the grp has never been initialized */ 20118a57d9d6SCurt Wohlgemuth if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 20128a57d9d6SCurt Wohlgemuth int ret = ext4_mb_init_group(ac->ac_sb, group); 20138a57d9d6SCurt Wohlgemuth if (ret) 20148a57d9d6SCurt Wohlgemuth return 0; 20158a57d9d6SCurt Wohlgemuth } 2016c9de560dSAlex Tomas 2017c9de560dSAlex Tomas fragments = grp->bb_fragments; 2018c9de560dSAlex Tomas if (fragments == 0) 2019c9de560dSAlex Tomas return 0; 2020c9de560dSAlex Tomas 2021c9de560dSAlex Tomas switch (cr) { 2022c9de560dSAlex Tomas case 0: 2023c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 2024c9de560dSAlex Tomas 2025a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 2026a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 2027a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 2028a4912123STheodore Ts'o ((group % flex_size) == 0)) 2029a4912123STheodore Ts'o return 0; 2030a4912123STheodore Ts'o 203140ae3487STheodore Ts'o if ((ac->ac_2order > ac->ac_sb->s_blocksize_bits+1) || 203240ae3487STheodore Ts'o (free / fragments) >= ac->ac_g_ex.fe_len) 203340ae3487STheodore Ts'o return 1; 203440ae3487STheodore Ts'o 203540ae3487STheodore Ts'o if (grp->bb_largest_free_order < ac->ac_2order) 203640ae3487STheodore Ts'o return 0; 203740ae3487STheodore Ts'o 2038c9de560dSAlex Tomas return 1; 2039c9de560dSAlex Tomas case 1: 2040c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 2041c9de560dSAlex Tomas return 1; 2042c9de560dSAlex Tomas break; 2043c9de560dSAlex Tomas case 2: 2044c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 2045c9de560dSAlex Tomas return 1; 2046c9de560dSAlex Tomas break; 2047c9de560dSAlex Tomas case 3: 2048c9de560dSAlex Tomas return 1; 2049c9de560dSAlex Tomas default: 2050c9de560dSAlex Tomas BUG(); 2051c9de560dSAlex Tomas } 2052c9de560dSAlex Tomas 2053c9de560dSAlex Tomas return 0; 2054c9de560dSAlex Tomas } 2055c9de560dSAlex Tomas 20564ddfef7bSEric Sandeen static noinline_for_stack int 20574ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 2058c9de560dSAlex Tomas { 20598df9675fSTheodore Ts'o ext4_group_t ngroups, group, i; 2060c9de560dSAlex Tomas int cr; 2061c9de560dSAlex Tomas int err = 0; 2062c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2063c9de560dSAlex Tomas struct super_block *sb; 2064c9de560dSAlex Tomas struct ext4_buddy e4b; 2065c9de560dSAlex Tomas 2066c9de560dSAlex Tomas sb = ac->ac_sb; 2067c9de560dSAlex Tomas sbi = EXT4_SB(sb); 20688df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 2069fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 207012e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 2071fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 2072fb0a387dSEric Sandeen 2073c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 2074c9de560dSAlex Tomas 2075c9de560dSAlex Tomas /* first, try the goal */ 2076c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 2077c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 2078c9de560dSAlex Tomas goto out; 2079c9de560dSAlex Tomas 2080c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2081c9de560dSAlex Tomas goto out; 2082c9de560dSAlex Tomas 2083c9de560dSAlex Tomas /* 2084c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 2085c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 2086c9de560dSAlex Tomas * try exact allocation using buddy. 2087c9de560dSAlex Tomas */ 2088c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 2089c9de560dSAlex Tomas ac->ac_2order = 0; 2090c9de560dSAlex Tomas /* 2091c9de560dSAlex Tomas * We search using buddy data only if the order of the request 2092c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 2093b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 2094c9de560dSAlex Tomas */ 2095c9de560dSAlex Tomas if (i >= sbi->s_mb_order2_reqs) { 2096c9de560dSAlex Tomas /* 2097c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2098c9de560dSAlex Tomas */ 2099c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 2100c9de560dSAlex Tomas ac->ac_2order = i - 1; 2101c9de560dSAlex Tomas } 2102c9de560dSAlex Tomas 21034ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 21044ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2105c9de560dSAlex Tomas /* TBD: may be hot point */ 2106c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2107c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2108c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2109c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2110c9de560dSAlex Tomas } 21114ba74d00STheodore Ts'o 2112c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2113c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2114c9de560dSAlex Tomas /* 2115c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2116c9de560dSAlex Tomas * cr == 3 try to get anything 2117c9de560dSAlex Tomas */ 2118c9de560dSAlex Tomas repeat: 2119c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2120c9de560dSAlex Tomas ac->ac_criteria = cr; 2121ed8f9c75SAneesh Kumar K.V /* 2122ed8f9c75SAneesh Kumar K.V * searching for the right group start 2123ed8f9c75SAneesh Kumar K.V * from the goal value specified 2124ed8f9c75SAneesh Kumar K.V */ 2125ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2126ed8f9c75SAneesh Kumar K.V 21278df9675fSTheodore Ts'o for (i = 0; i < ngroups; group++, i++) { 21282ed5724dSTheodore Ts'o cond_resched(); 2129e6155736SLachlan McIlroy /* 2130e6155736SLachlan McIlroy * Artificially restricted ngroups for non-extent 2131e6155736SLachlan McIlroy * files makes group > ngroups possible on first loop. 2132e6155736SLachlan McIlroy */ 2133e6155736SLachlan McIlroy if (group >= ngroups) 2134c9de560dSAlex Tomas group = 0; 2135c9de560dSAlex Tomas 21368a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 21378a57d9d6SCurt Wohlgemuth if (!ext4_mb_good_group(ac, group, cr)) 2138c9de560dSAlex Tomas continue; 2139c9de560dSAlex Tomas 2140c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2141c9de560dSAlex Tomas if (err) 2142c9de560dSAlex Tomas goto out; 2143c9de560dSAlex Tomas 2144c9de560dSAlex Tomas ext4_lock_group(sb, group); 21458a57d9d6SCurt Wohlgemuth 21468a57d9d6SCurt Wohlgemuth /* 21478a57d9d6SCurt Wohlgemuth * We need to check again after locking the 21488a57d9d6SCurt Wohlgemuth * block group 21498a57d9d6SCurt Wohlgemuth */ 2150c9de560dSAlex Tomas if (!ext4_mb_good_group(ac, group, cr)) { 2151c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2152e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2153c9de560dSAlex Tomas continue; 2154c9de560dSAlex Tomas } 2155c9de560dSAlex Tomas 2156c9de560dSAlex Tomas ac->ac_groups_scanned++; 215740ae3487STheodore Ts'o if (cr == 0 && ac->ac_2order < sb->s_blocksize_bits+2) 2158c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2159506bf2d8SEric Sandeen else if (cr == 1 && sbi->s_stripe && 2160506bf2d8SEric Sandeen !(ac->ac_g_ex.fe_len % sbi->s_stripe)) 2161c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2162c9de560dSAlex Tomas else 2163c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2164c9de560dSAlex Tomas 2165c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2166e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2167c9de560dSAlex Tomas 2168c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2169c9de560dSAlex Tomas break; 2170c9de560dSAlex Tomas } 2171c9de560dSAlex Tomas } 2172c9de560dSAlex Tomas 2173c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2174c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2175c9de560dSAlex Tomas /* 2176c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2177c9de560dSAlex Tomas * the best chunk we've found so far 2178c9de560dSAlex Tomas */ 2179c9de560dSAlex Tomas 2180c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2181c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2182c9de560dSAlex Tomas /* 2183c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2184c9de560dSAlex Tomas * The only thing we can do is just take first 2185c9de560dSAlex Tomas * found block(s) 2186c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2187c9de560dSAlex Tomas */ 2188c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2189c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2190c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2191c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2192c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2193c9de560dSAlex Tomas cr = 3; 2194c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2195c9de560dSAlex Tomas goto repeat; 2196c9de560dSAlex Tomas } 2197c9de560dSAlex Tomas } 2198c9de560dSAlex Tomas out: 2199c9de560dSAlex Tomas return err; 2200c9de560dSAlex Tomas } 2201c9de560dSAlex Tomas 2202c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2203c9de560dSAlex Tomas { 2204c9de560dSAlex Tomas struct super_block *sb = seq->private; 2205c9de560dSAlex Tomas ext4_group_t group; 2206c9de560dSAlex Tomas 22078df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2208c9de560dSAlex Tomas return NULL; 2209c9de560dSAlex Tomas group = *pos + 1; 2210a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2211c9de560dSAlex Tomas } 2212c9de560dSAlex Tomas 2213c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2214c9de560dSAlex Tomas { 2215c9de560dSAlex Tomas struct super_block *sb = seq->private; 2216c9de560dSAlex Tomas ext4_group_t group; 2217c9de560dSAlex Tomas 2218c9de560dSAlex Tomas ++*pos; 22198df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2220c9de560dSAlex Tomas return NULL; 2221c9de560dSAlex Tomas group = *pos + 1; 2222a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2223c9de560dSAlex Tomas } 2224c9de560dSAlex Tomas 2225c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2226c9de560dSAlex Tomas { 2227c9de560dSAlex Tomas struct super_block *sb = seq->private; 2228a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2229c9de560dSAlex Tomas int i; 22301c8457caSAditya Kali int err, buddy_loaded = 0; 2231c9de560dSAlex Tomas struct ext4_buddy e4b; 22321c8457caSAditya Kali struct ext4_group_info *grinfo; 2233c9de560dSAlex Tomas struct sg { 2234c9de560dSAlex Tomas struct ext4_group_info info; 2235a36b4498SEric Sandeen ext4_grpblk_t counters[16]; 2236c9de560dSAlex Tomas } sg; 2237c9de560dSAlex Tomas 2238c9de560dSAlex Tomas group--; 2239c9de560dSAlex Tomas if (group == 0) 2240c9de560dSAlex Tomas seq_printf(seq, "#%-5s: %-5s %-5s %-5s " 2241c9de560dSAlex Tomas "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s " 2242c9de560dSAlex Tomas "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n", 2243c9de560dSAlex Tomas "group", "free", "frags", "first", 2244c9de560dSAlex Tomas "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6", 2245c9de560dSAlex Tomas "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13"); 2246c9de560dSAlex Tomas 2247c9de560dSAlex Tomas i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2248c9de560dSAlex Tomas sizeof(struct ext4_group_info); 22491c8457caSAditya Kali grinfo = ext4_get_group_info(sb, group); 22501c8457caSAditya Kali /* Load the group info in memory only if not already loaded. */ 22511c8457caSAditya Kali if (unlikely(EXT4_MB_GRP_NEED_INIT(grinfo))) { 2252c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2253c9de560dSAlex Tomas if (err) { 2254a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2255c9de560dSAlex Tomas return 0; 2256c9de560dSAlex Tomas } 22571c8457caSAditya Kali buddy_loaded = 1; 22581c8457caSAditya Kali } 22591c8457caSAditya Kali 2260c9de560dSAlex Tomas memcpy(&sg, ext4_get_group_info(sb, group), i); 22611c8457caSAditya Kali 22621c8457caSAditya Kali if (buddy_loaded) 2263e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2264c9de560dSAlex Tomas 2265a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2266c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2267c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 2268c9de560dSAlex Tomas seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ? 2269c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2270c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2271c9de560dSAlex Tomas 2272c9de560dSAlex Tomas return 0; 2273c9de560dSAlex Tomas } 2274c9de560dSAlex Tomas 2275c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2276c9de560dSAlex Tomas { 2277c9de560dSAlex Tomas } 2278c9de560dSAlex Tomas 22797f1346a9STobias Klauser static const struct seq_operations ext4_mb_seq_groups_ops = { 2280c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2281c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2282c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2283c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2284c9de560dSAlex Tomas }; 2285c9de560dSAlex Tomas 2286c9de560dSAlex Tomas static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file) 2287c9de560dSAlex Tomas { 2288d9dda78bSAl Viro struct super_block *sb = PDE_DATA(inode); 2289c9de560dSAlex Tomas int rc; 2290c9de560dSAlex Tomas 2291c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_groups_ops); 2292c9de560dSAlex Tomas if (rc == 0) { 2293a271fe85SJoe Perches struct seq_file *m = file->private_data; 2294c9de560dSAlex Tomas m->private = sb; 2295c9de560dSAlex Tomas } 2296c9de560dSAlex Tomas return rc; 2297c9de560dSAlex Tomas 2298c9de560dSAlex Tomas } 2299c9de560dSAlex Tomas 23007f1346a9STobias Klauser static const struct file_operations ext4_mb_seq_groups_fops = { 2301c9de560dSAlex Tomas .owner = THIS_MODULE, 2302c9de560dSAlex Tomas .open = ext4_mb_seq_groups_open, 2303c9de560dSAlex Tomas .read = seq_read, 2304c9de560dSAlex Tomas .llseek = seq_lseek, 2305c9de560dSAlex Tomas .release = seq_release, 2306c9de560dSAlex Tomas }; 2307c9de560dSAlex Tomas 2308fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 2309fb1813f4SCurt Wohlgemuth { 2310fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 2311fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 2312fb1813f4SCurt Wohlgemuth 2313fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 2314fb1813f4SCurt Wohlgemuth return cachep; 2315fb1813f4SCurt Wohlgemuth } 23165f21b0e6SFrederic Bohe 231728623c2fSTheodore Ts'o /* 231828623c2fSTheodore Ts'o * Allocate the top-level s_group_info array for the specified number 231928623c2fSTheodore Ts'o * of groups 232028623c2fSTheodore Ts'o */ 232128623c2fSTheodore Ts'o int ext4_mb_alloc_groupinfo(struct super_block *sb, ext4_group_t ngroups) 232228623c2fSTheodore Ts'o { 232328623c2fSTheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 232428623c2fSTheodore Ts'o unsigned size; 232528623c2fSTheodore Ts'o struct ext4_group_info ***new_groupinfo; 232628623c2fSTheodore Ts'o 232728623c2fSTheodore Ts'o size = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 1) >> 232828623c2fSTheodore Ts'o EXT4_DESC_PER_BLOCK_BITS(sb); 232928623c2fSTheodore Ts'o if (size <= sbi->s_group_info_size) 233028623c2fSTheodore Ts'o return 0; 233128623c2fSTheodore Ts'o 233228623c2fSTheodore Ts'o size = roundup_pow_of_two(sizeof(*sbi->s_group_info) * size); 233328623c2fSTheodore Ts'o new_groupinfo = ext4_kvzalloc(size, GFP_KERNEL); 233428623c2fSTheodore Ts'o if (!new_groupinfo) { 233528623c2fSTheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 233628623c2fSTheodore Ts'o return -ENOMEM; 233728623c2fSTheodore Ts'o } 233828623c2fSTheodore Ts'o if (sbi->s_group_info) { 233928623c2fSTheodore Ts'o memcpy(new_groupinfo, sbi->s_group_info, 234028623c2fSTheodore Ts'o sbi->s_group_info_size * sizeof(*sbi->s_group_info)); 234128623c2fSTheodore Ts'o ext4_kvfree(sbi->s_group_info); 234228623c2fSTheodore Ts'o } 234328623c2fSTheodore Ts'o sbi->s_group_info = new_groupinfo; 234428623c2fSTheodore Ts'o sbi->s_group_info_size = size / sizeof(*sbi->s_group_info); 234528623c2fSTheodore Ts'o ext4_debug("allocated s_groupinfo array for %d meta_bg's\n", 234628623c2fSTheodore Ts'o sbi->s_group_info_size); 234728623c2fSTheodore Ts'o return 0; 234828623c2fSTheodore Ts'o } 234928623c2fSTheodore Ts'o 23505f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 2351920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 23525f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 23535f21b0e6SFrederic Bohe { 2354fb1813f4SCurt Wohlgemuth int i; 23555f21b0e6SFrederic Bohe int metalen = 0; 23565f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 23575f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 2358fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 23595f21b0e6SFrederic Bohe 23605f21b0e6SFrederic Bohe /* 23615f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 23625f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 23635f21b0e6SFrederic Bohe * to ext4_group_info structures 23645f21b0e6SFrederic Bohe */ 23655f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 23665f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 23675f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 23685f21b0e6SFrederic Bohe meta_group_info = kmalloc(metalen, GFP_KERNEL); 23695f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 23707f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate mem " 23719d8b9ec4STheodore Ts'o "for a buddy group"); 23725f21b0e6SFrederic Bohe goto exit_meta_group_info; 23735f21b0e6SFrederic Bohe } 23745f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = 23755f21b0e6SFrederic Bohe meta_group_info; 23765f21b0e6SFrederic Bohe } 23775f21b0e6SFrederic Bohe 23785f21b0e6SFrederic Bohe meta_group_info = 23795f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]; 23805f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 23815f21b0e6SFrederic Bohe 238285556c9aSWei Yongjun meta_group_info[i] = kmem_cache_zalloc(cachep, GFP_KERNEL); 23835f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 23847f6a11e7SJoe Perches ext4_msg(sb, KERN_ERR, "can't allocate buddy mem"); 23855f21b0e6SFrederic Bohe goto exit_group_info; 23865f21b0e6SFrederic Bohe } 23875f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 23885f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 23895f21b0e6SFrederic Bohe 23905f21b0e6SFrederic Bohe /* 23915f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 23925f21b0e6SFrederic Bohe * empty groups without initialization 23935f21b0e6SFrederic Bohe */ 23945f21b0e6SFrederic Bohe if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 23955f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2396cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, group, desc); 23975f21b0e6SFrederic Bohe } else { 23985f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2399021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, desc); 24005f21b0e6SFrederic Bohe } 24015f21b0e6SFrederic Bohe 24025f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 2403920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 240464e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 24058a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 24065f21b0e6SFrederic Bohe 24075f21b0e6SFrederic Bohe #ifdef DOUBLE_CHECK 24085f21b0e6SFrederic Bohe { 24095f21b0e6SFrederic Bohe struct buffer_head *bh; 24105f21b0e6SFrederic Bohe meta_group_info[i]->bb_bitmap = 24115f21b0e6SFrederic Bohe kmalloc(sb->s_blocksize, GFP_KERNEL); 24125f21b0e6SFrederic Bohe BUG_ON(meta_group_info[i]->bb_bitmap == NULL); 24135f21b0e6SFrederic Bohe bh = ext4_read_block_bitmap(sb, group); 24145f21b0e6SFrederic Bohe BUG_ON(bh == NULL); 24155f21b0e6SFrederic Bohe memcpy(meta_group_info[i]->bb_bitmap, bh->b_data, 24165f21b0e6SFrederic Bohe sb->s_blocksize); 24175f21b0e6SFrederic Bohe put_bh(bh); 24185f21b0e6SFrederic Bohe } 24195f21b0e6SFrederic Bohe #endif 24205f21b0e6SFrederic Bohe 24215f21b0e6SFrederic Bohe return 0; 24225f21b0e6SFrederic Bohe 24235f21b0e6SFrederic Bohe exit_group_info: 24245f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 2425caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 24265f21b0e6SFrederic Bohe kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]); 2427caaf7a29STao Ma sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = NULL; 2428caaf7a29STao Ma } 24295f21b0e6SFrederic Bohe exit_meta_group_info: 24305f21b0e6SFrederic Bohe return -ENOMEM; 24315f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 24325f21b0e6SFrederic Bohe 2433c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2434c9de560dSAlex Tomas { 24358df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2436c9de560dSAlex Tomas ext4_group_t i; 2437c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 243828623c2fSTheodore Ts'o int err; 24395f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 2440fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 2441c9de560dSAlex Tomas 244228623c2fSTheodore Ts'o err = ext4_mb_alloc_groupinfo(sb, ngroups); 244328623c2fSTheodore Ts'o if (err) 244428623c2fSTheodore Ts'o return err; 24455f21b0e6SFrederic Bohe 2446c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2447c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 24489d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 2449c9de560dSAlex Tomas goto err_freesgi; 2450c9de560dSAlex Tomas } 245148e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 245248e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 245348e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 245448e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 245548e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 2456c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 24578df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2458c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2459c9de560dSAlex Tomas if (desc == NULL) { 24609d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 2461c9de560dSAlex Tomas goto err_freebuddy; 2462c9de560dSAlex Tomas } 24635f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 24645f21b0e6SFrederic Bohe goto err_freebuddy; 2465c9de560dSAlex Tomas } 2466c9de560dSAlex Tomas 2467c9de560dSAlex Tomas return 0; 2468c9de560dSAlex Tomas 2469c9de560dSAlex Tomas err_freebuddy: 2470fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2471f1fa3342SRoel Kluin while (i-- > 0) 2472fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, ext4_get_group_info(sb, i)); 247328623c2fSTheodore Ts'o i = sbi->s_group_info_size; 2474f1fa3342SRoel Kluin while (i-- > 0) 2475c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2476c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2477c9de560dSAlex Tomas err_freesgi: 2478f18a5f21STheodore Ts'o ext4_kvfree(sbi->s_group_info); 2479c9de560dSAlex Tomas return -ENOMEM; 2480c9de560dSAlex Tomas } 2481c9de560dSAlex Tomas 24822892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 24832892c15dSEric Sandeen { 24842892c15dSEric Sandeen int i; 24852892c15dSEric Sandeen 24862892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 24872892c15dSEric Sandeen if (ext4_groupinfo_caches[i]) 24882892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 24892892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 24902892c15dSEric Sandeen } 24912892c15dSEric Sandeen } 24922892c15dSEric Sandeen 24932892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 24942892c15dSEric Sandeen { 24952892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 24962892c15dSEric Sandeen int slab_size; 24972892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 24982892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 24992892c15dSEric Sandeen struct kmem_cache *cachep; 25002892c15dSEric Sandeen 25012892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 25022892c15dSEric Sandeen return -EINVAL; 25032892c15dSEric Sandeen 25042892c15dSEric Sandeen if (unlikely(cache_index < 0)) 25052892c15dSEric Sandeen cache_index = 0; 25062892c15dSEric Sandeen 25072892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 25082892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 25092892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 25102892c15dSEric Sandeen return 0; /* Already created */ 25112892c15dSEric Sandeen } 25122892c15dSEric Sandeen 25132892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 25142892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 25152892c15dSEric Sandeen 25162892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 25172892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 25182892c15dSEric Sandeen NULL); 25192892c15dSEric Sandeen 2520823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 2521823ba01fSTao Ma 25222892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 25232892c15dSEric Sandeen if (!cachep) { 25249d8b9ec4STheodore Ts'o printk(KERN_EMERG 25259d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 25262892c15dSEric Sandeen return -ENOMEM; 25272892c15dSEric Sandeen } 25282892c15dSEric Sandeen 25292892c15dSEric Sandeen return 0; 25302892c15dSEric Sandeen } 25312892c15dSEric Sandeen 25329d99012fSAkira Fujita int ext4_mb_init(struct super_block *sb) 2533c9de560dSAlex Tomas { 2534c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 25356be2ded1SAneesh Kumar K.V unsigned i, j; 2536c9de560dSAlex Tomas unsigned offset; 2537c9de560dSAlex Tomas unsigned max; 253874767c5aSShen Feng int ret; 2539c9de560dSAlex Tomas 25401927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); 2541c9de560dSAlex Tomas 2542c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2543c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2544fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2545fb1813f4SCurt Wohlgemuth goto out; 2546c9de560dSAlex Tomas } 2547ff7ef329SYasunori Goto 25481927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); 2549c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2550c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2551fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2552fb1813f4SCurt Wohlgemuth goto out; 2553fb1813f4SCurt Wohlgemuth } 2554fb1813f4SCurt Wohlgemuth 25552892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 25562892c15dSEric Sandeen if (ret < 0) 2557fb1813f4SCurt Wohlgemuth goto out; 2558c9de560dSAlex Tomas 2559c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2560c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2561c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2562c9de560dSAlex Tomas 2563c9de560dSAlex Tomas i = 1; 2564c9de560dSAlex Tomas offset = 0; 2565c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2566c9de560dSAlex Tomas do { 2567c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2568c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2569c9de560dSAlex Tomas offset += 1 << (sb->s_blocksize_bits - i); 2570c9de560dSAlex Tomas max = max >> 1; 2571c9de560dSAlex Tomas i++; 2572c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2573c9de560dSAlex Tomas 2574c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2575c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2576c9de560dSAlex Tomas 2577c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2578c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2579c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2580c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2581c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 258227baebb8STheodore Ts'o /* 258327baebb8STheodore Ts'o * The default group preallocation is 512, which for 4k block 258427baebb8STheodore Ts'o * sizes translates to 2 megabytes. However for bigalloc file 258527baebb8STheodore Ts'o * systems, this is probably too big (i.e, if the cluster size 258627baebb8STheodore Ts'o * is 1 megabyte, then group preallocation size becomes half a 258727baebb8STheodore Ts'o * gigabyte!). As a default, we will keep a two megabyte 258827baebb8STheodore Ts'o * group pralloc size for cluster sizes up to 64k, and after 258927baebb8STheodore Ts'o * that, we will force a minimum group preallocation size of 259027baebb8STheodore Ts'o * 32 clusters. This translates to 8 megs when the cluster 259127baebb8STheodore Ts'o * size is 256k, and 32 megs when the cluster size is 1 meg, 259227baebb8STheodore Ts'o * which seems reasonable as a default. 259327baebb8STheodore Ts'o */ 259427baebb8STheodore Ts'o sbi->s_mb_group_prealloc = max(MB_DEFAULT_GROUP_PREALLOC >> 259527baebb8STheodore Ts'o sbi->s_cluster_bits, 32); 2596d7a1fee1SDan Ehrenberg /* 2597d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 2598d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 2599d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 2600d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 2601d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 2602d7a1fee1SDan Ehrenberg * the stripes. 2603d7a1fee1SDan Ehrenberg */ 2604d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 2605d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 2606d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc, sbi->s_stripe); 2607d7a1fee1SDan Ehrenberg } 2608c9de560dSAlex Tomas 2609730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 2610c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2611fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 26127aa0baeaSTao Ma goto out_free_groupinfo_slab; 2613c9de560dSAlex Tomas } 2614730c213cSEric Sandeen for_each_possible_cpu(i) { 2615c9de560dSAlex Tomas struct ext4_locality_group *lg; 2616730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 2617c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 26186be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 26196be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 2620c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2621c9de560dSAlex Tomas } 2622c9de560dSAlex Tomas 262379a77c5aSYu Jian /* init file for buddy data */ 262479a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 26257aa0baeaSTao Ma if (ret != 0) 26267aa0baeaSTao Ma goto out_free_locality_groups; 262779a77c5aSYu Jian 2628296c355cSTheodore Ts'o if (sbi->s_proc) 2629296c355cSTheodore Ts'o proc_create_data("mb_groups", S_IRUGO, sbi->s_proc, 2630296c355cSTheodore Ts'o &ext4_mb_seq_groups_fops, sb); 2631c9de560dSAlex Tomas 26327aa0baeaSTao Ma return 0; 26337aa0baeaSTao Ma 26347aa0baeaSTao Ma out_free_locality_groups: 26357aa0baeaSTao Ma free_percpu(sbi->s_locality_groups); 26367aa0baeaSTao Ma sbi->s_locality_groups = NULL; 26377aa0baeaSTao Ma out_free_groupinfo_slab: 26387aa0baeaSTao Ma ext4_groupinfo_destroy_slabs(); 2639fb1813f4SCurt Wohlgemuth out: 2640fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 26417aa0baeaSTao Ma sbi->s_mb_offsets = NULL; 2642fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 26437aa0baeaSTao Ma sbi->s_mb_maxs = NULL; 2644fb1813f4SCurt Wohlgemuth return ret; 2645c9de560dSAlex Tomas } 2646c9de560dSAlex Tomas 2647955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 2648c9de560dSAlex Tomas static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2649c9de560dSAlex Tomas { 2650c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2651c9de560dSAlex Tomas struct list_head *cur, *tmp; 2652c9de560dSAlex Tomas int count = 0; 2653c9de560dSAlex Tomas 2654c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2655c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2656c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2657c9de560dSAlex Tomas count++; 2658688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 2659c9de560dSAlex Tomas } 2660c9de560dSAlex Tomas if (count) 26616ba495e9STheodore Ts'o mb_debug(1, "mballoc: %u PAs left\n", count); 2662c9de560dSAlex Tomas 2663c9de560dSAlex Tomas } 2664c9de560dSAlex Tomas 2665c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2666c9de560dSAlex Tomas { 26678df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2668c9de560dSAlex Tomas ext4_group_t i; 2669c9de560dSAlex Tomas int num_meta_group_infos; 2670c9de560dSAlex Tomas struct ext4_group_info *grinfo; 2671c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2672fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2673c9de560dSAlex Tomas 267495599968SSalman Qazi if (sbi->s_proc) 267595599968SSalman Qazi remove_proc_entry("mb_groups", sbi->s_proc); 267695599968SSalman Qazi 2677c9de560dSAlex Tomas if (sbi->s_group_info) { 26788df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2679c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2680c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2681c9de560dSAlex Tomas kfree(grinfo->bb_bitmap); 2682c9de560dSAlex Tomas #endif 2683c9de560dSAlex Tomas ext4_lock_group(sb, i); 2684c9de560dSAlex Tomas ext4_mb_cleanup_pa(grinfo); 2685c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2686fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 2687c9de560dSAlex Tomas } 26888df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 2689c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2690c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2691c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2692c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2693f18a5f21STheodore Ts'o ext4_kvfree(sbi->s_group_info); 2694c9de560dSAlex Tomas } 2695c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2696c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2697c9de560dSAlex Tomas if (sbi->s_buddy_cache) 2698c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2699c9de560dSAlex Tomas if (sbi->s_mb_stats) { 27009d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27019d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 2702c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2703c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2704c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 27059d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27069d8b9ec4STheodore Ts'o "mballoc: %u extents scanned, %u goal hits, " 27079d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 2708c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 2709c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 2710c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 2711c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 2712c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 27139d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27149d8b9ec4STheodore Ts'o "mballoc: %lu generated and it took %Lu", 2715ced156e4STao Ma sbi->s_mb_buddies_generated, 2716c9de560dSAlex Tomas sbi->s_mb_generation_time); 27179d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 27189d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 2719c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 2720c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 2721c9de560dSAlex Tomas } 2722c9de560dSAlex Tomas 2723730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 2724c9de560dSAlex Tomas 2725c9de560dSAlex Tomas return 0; 2726c9de560dSAlex Tomas } 2727c9de560dSAlex Tomas 272877ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 272984130193STheodore Ts'o ext4_group_t block_group, ext4_grpblk_t cluster, int count) 27305c521830SJiaying Zhang { 27315c521830SJiaying Zhang ext4_fsblk_t discard_block; 27325c521830SJiaying Zhang 273384130193STheodore Ts'o discard_block = (EXT4_C2B(EXT4_SB(sb), cluster) + 273484130193STheodore Ts'o ext4_group_first_block_no(sb, block_group)); 273584130193STheodore Ts'o count = EXT4_C2B(EXT4_SB(sb), count); 27365c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 27375c521830SJiaying Zhang (unsigned long long) discard_block, count); 273893259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 27395c521830SJiaying Zhang } 27405c521830SJiaying Zhang 27413e624fc7STheodore Ts'o /* 27423e624fc7STheodore Ts'o * This function is called by the jbd2 layer once the commit has finished, 27433e624fc7STheodore Ts'o * so we know we can free the blocks that were released with that commit. 27443e624fc7STheodore Ts'o */ 274518aadd47SBobi Jam static void ext4_free_data_callback(struct super_block *sb, 274618aadd47SBobi Jam struct ext4_journal_cb_entry *jce, 274718aadd47SBobi Jam int rc) 2748c9de560dSAlex Tomas { 274918aadd47SBobi Jam struct ext4_free_data *entry = (struct ext4_free_data *)jce; 2750c9de560dSAlex Tomas struct ext4_buddy e4b; 2751c894058dSAneesh Kumar K.V struct ext4_group_info *db; 2752d9f34504STheodore Ts'o int err, count = 0, count2 = 0; 2753c9de560dSAlex Tomas 27546ba495e9STheodore Ts'o mb_debug(1, "gonna free %u blocks in group %u (0x%p):", 275518aadd47SBobi Jam entry->efd_count, entry->efd_group, entry); 2756c9de560dSAlex Tomas 2757d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 2758d71c1ae2SLukas Czerner err = ext4_issue_discard(sb, entry->efd_group, 2759d71c1ae2SLukas Czerner entry->efd_start_cluster, 2760d71c1ae2SLukas Czerner entry->efd_count); 2761d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 2762d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 2763d71c1ae2SLukas Czerner " group:%d block:%d count:%d failed" 2764d71c1ae2SLukas Czerner " with %d", entry->efd_group, 2765d71c1ae2SLukas Czerner entry->efd_start_cluster, 2766d71c1ae2SLukas Czerner entry->efd_count, err); 2767d71c1ae2SLukas Czerner } 2768b90f6870STheodore Ts'o 276918aadd47SBobi Jam err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b); 2770c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 2771c9de560dSAlex Tomas BUG_ON(err != 0); 2772c9de560dSAlex Tomas 277318aadd47SBobi Jam 2774c894058dSAneesh Kumar K.V db = e4b.bd_info; 2775c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 277618aadd47SBobi Jam count += entry->efd_count; 2777c9de560dSAlex Tomas count2++; 277818aadd47SBobi Jam ext4_lock_group(sb, entry->efd_group); 2779c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 278018aadd47SBobi Jam rb_erase(&entry->efd_node, &(db->bb_free_root)); 278118aadd47SBobi Jam mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count); 2782c9de560dSAlex Tomas 27833d56b8d2STao Ma /* 27843d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 27853d56b8d2STao Ma * ext4_trim_fs can trim it. 27863d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 27873d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 27883d56b8d2STao Ma */ 27893d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 27903d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 27913d56b8d2STao Ma 2792c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 2793c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 2794c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 2795c894058dSAneesh Kumar K.V */ 2796c9de560dSAlex Tomas page_cache_release(e4b.bd_buddy_page); 2797c9de560dSAlex Tomas page_cache_release(e4b.bd_bitmap_page); 2798c894058dSAneesh Kumar K.V } 279918aadd47SBobi Jam ext4_unlock_group(sb, entry->efd_group); 280018aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 2801e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2802c9de560dSAlex Tomas 28036ba495e9STheodore Ts'o mb_debug(1, "freed %u blocks in %u structures\n", count, count2); 2804c9de560dSAlex Tomas } 2805c9de560dSAlex Tomas 28065dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 2807c9de560dSAlex Tomas { 280816828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 280916828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2810c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 2811c9de560dSAlex Tomas return -ENOMEM; 2812c9de560dSAlex Tomas 281316828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 281416828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2815256bdb49SEric Sandeen if (ext4_ac_cachep == NULL) { 2816256bdb49SEric Sandeen kmem_cache_destroy(ext4_pspace_cachep); 2817256bdb49SEric Sandeen return -ENOMEM; 2818256bdb49SEric Sandeen } 2819c894058dSAneesh Kumar K.V 282018aadd47SBobi Jam ext4_free_data_cachep = KMEM_CACHE(ext4_free_data, 282116828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 282218aadd47SBobi Jam if (ext4_free_data_cachep == NULL) { 2823c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_pspace_cachep); 2824c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_ac_cachep); 2825c894058dSAneesh Kumar K.V return -ENOMEM; 2826c894058dSAneesh Kumar K.V } 2827c9de560dSAlex Tomas return 0; 2828c9de560dSAlex Tomas } 2829c9de560dSAlex Tomas 28305dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 2831c9de560dSAlex Tomas { 28323e03f9caSJesper Dangaard Brouer /* 28333e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 28343e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 28353e03f9caSJesper Dangaard Brouer */ 28363e03f9caSJesper Dangaard Brouer rcu_barrier(); 2837c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 2838256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 283918aadd47SBobi Jam kmem_cache_destroy(ext4_free_data_cachep); 28402892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 2841c9de560dSAlex Tomas } 2842c9de560dSAlex Tomas 2843c9de560dSAlex Tomas 2844c9de560dSAlex Tomas /* 284573b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 2846c9de560dSAlex Tomas * Returns 0 if success or error code 2847c9de560dSAlex Tomas */ 28484ddfef7bSEric Sandeen static noinline_for_stack int 28494ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 285053accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 2851c9de560dSAlex Tomas { 2852c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 2853c9de560dSAlex Tomas struct ext4_group_desc *gdp; 2854c9de560dSAlex Tomas struct buffer_head *gdp_bh; 2855c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2856c9de560dSAlex Tomas struct super_block *sb; 2857c9de560dSAlex Tomas ext4_fsblk_t block; 2858519deca0SAneesh Kumar K.V int err, len; 2859c9de560dSAlex Tomas 2860c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 2861c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 2862c9de560dSAlex Tomas 2863c9de560dSAlex Tomas sb = ac->ac_sb; 2864c9de560dSAlex Tomas sbi = EXT4_SB(sb); 2865c9de560dSAlex Tomas 2866c9de560dSAlex Tomas err = -EIO; 2867574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 2868c9de560dSAlex Tomas if (!bitmap_bh) 2869c9de560dSAlex Tomas goto out_err; 2870c9de560dSAlex Tomas 2871c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 2872c9de560dSAlex Tomas if (err) 2873c9de560dSAlex Tomas goto out_err; 2874c9de560dSAlex Tomas 2875c9de560dSAlex Tomas err = -EIO; 2876c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 2877c9de560dSAlex Tomas if (!gdp) 2878c9de560dSAlex Tomas goto out_err; 2879c9de560dSAlex Tomas 2880a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 2881021b65bbSTheodore Ts'o ext4_free_group_clusters(sb, gdp)); 288203cddb80SAneesh Kumar K.V 2883c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 2884c9de560dSAlex Tomas if (err) 2885c9de560dSAlex Tomas goto out_err; 2886c9de560dSAlex Tomas 2887bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 2888c9de560dSAlex Tomas 288953accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 28906fd058f7STheodore Ts'o if (!ext4_data_block_valid(sbi, block, len)) { 289112062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 28921084f252STheodore Ts'o "fs metadata", block, block+len); 2893519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 2894519deca0SAneesh Kumar K.V * Fix the bitmap and repeat the block allocation 2895519deca0SAneesh Kumar K.V * We leak some of the blocks here. 2896519deca0SAneesh Kumar K.V */ 2897955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 2898c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 2899519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 2900955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 29010390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 2902519deca0SAneesh Kumar K.V if (!err) 2903519deca0SAneesh Kumar K.V err = -EAGAIN; 2904519deca0SAneesh Kumar K.V goto out_err; 2905c9de560dSAlex Tomas } 2906955ce5f5SAneesh Kumar K.V 2907955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 2908c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 2909c9de560dSAlex Tomas { 2910c9de560dSAlex Tomas int i; 2911c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 2912c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 2913c9de560dSAlex Tomas bitmap_bh->b_data)); 2914c9de560dSAlex Tomas } 2915c9de560dSAlex Tomas } 2916c9de560dSAlex Tomas #endif 2917c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 2918c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 2919c9de560dSAlex Tomas if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 2920c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 2921021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, 2922cff1dfd7STheodore Ts'o ext4_free_clusters_after_init(sb, 2923560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 2924c9de560dSAlex Tomas } 2925021b65bbSTheodore Ts'o len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len; 2926021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, len); 292779f1ba49STao Ma ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh); 2928feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp); 2929955ce5f5SAneesh Kumar K.V 2930955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 293157042651STheodore Ts'o percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len); 2932d2a17637SMingming Cao /* 29336bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 2934d2a17637SMingming Cao */ 29356bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 29366bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 293757042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 293857042651STheodore Ts'o reserv_clstrs); 2939c9de560dSAlex Tomas 2940772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 2941772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 2942772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 294390ba983fSTheodore Ts'o atomic64_sub(ac->ac_b_ex.fe_len, 294424aaa8efSTheodore Ts'o &sbi->s_flex_groups[flex_group].free_clusters); 2945772cb7c8SJose R. Santos } 2946772cb7c8SJose R. Santos 29470390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 2948c9de560dSAlex Tomas if (err) 2949c9de560dSAlex Tomas goto out_err; 29500390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 2951c9de560dSAlex Tomas 2952c9de560dSAlex Tomas out_err: 295342a10addSAneesh Kumar K.V brelse(bitmap_bh); 2954c9de560dSAlex Tomas return err; 2955c9de560dSAlex Tomas } 2956c9de560dSAlex Tomas 2957c9de560dSAlex Tomas /* 2958c9de560dSAlex Tomas * here we normalize request for locality group 2959d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 2960d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 2961d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 2962b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 2963c9de560dSAlex Tomas * 2964c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 2965c9de560dSAlex Tomas */ 2966c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 2967c9de560dSAlex Tomas { 2968c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2969c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 2970c9de560dSAlex Tomas 2971c9de560dSAlex Tomas BUG_ON(lg == NULL); 2972c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 29736ba495e9STheodore Ts'o mb_debug(1, "#%u: goal %u blocks for locality group\n", 2974c9de560dSAlex Tomas current->pid, ac->ac_g_ex.fe_len); 2975c9de560dSAlex Tomas } 2976c9de560dSAlex Tomas 2977c9de560dSAlex Tomas /* 2978c9de560dSAlex Tomas * Normalization means making request better in terms of 2979c9de560dSAlex Tomas * size and alignment 2980c9de560dSAlex Tomas */ 29814ddfef7bSEric Sandeen static noinline_for_stack void 29824ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 2983c9de560dSAlex Tomas struct ext4_allocation_request *ar) 2984c9de560dSAlex Tomas { 298553accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2986c9de560dSAlex Tomas int bsbits, max; 2987c9de560dSAlex Tomas ext4_lblk_t end; 29881592d2c5SCurt Wohlgemuth loff_t size, start_off; 29891592d2c5SCurt Wohlgemuth loff_t orig_size __maybe_unused; 29905a0790c2SAndi Kleen ext4_lblk_t start; 2991c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 29929a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 2993c9de560dSAlex Tomas 2994c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 2995c9de560dSAlex Tomas do not need preallocation */ 2996c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 2997c9de560dSAlex Tomas return; 2998c9de560dSAlex Tomas 2999c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3000c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3001c9de560dSAlex Tomas return; 3002c9de560dSAlex Tomas 3003c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3004c9de560dSAlex Tomas * required (it's a tail, for example) */ 3005c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3006c9de560dSAlex Tomas return; 3007c9de560dSAlex Tomas 3008c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3009c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3010c9de560dSAlex Tomas return ; 3011c9de560dSAlex Tomas } 3012c9de560dSAlex Tomas 3013c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3014c9de560dSAlex Tomas 3015c9de560dSAlex Tomas /* first, let's learn actual file size 3016c9de560dSAlex Tomas * given current request is allocated */ 301753accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 3018c9de560dSAlex Tomas size = size << bsbits; 3019c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3020c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 30215a0790c2SAndi Kleen orig_size = size; 3022c9de560dSAlex Tomas 30231930479cSValerie Clement /* max size of free chunks */ 30241930479cSValerie Clement max = 2 << bsbits; 3025c9de560dSAlex Tomas 30261930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 30271930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 3028c9de560dSAlex Tomas 3029c9de560dSAlex Tomas /* first, try to predict filesize */ 3030c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3031c9de560dSAlex Tomas start_off = 0; 3032c9de560dSAlex Tomas if (size <= 16 * 1024) { 3033c9de560dSAlex Tomas size = 16 * 1024; 3034c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3035c9de560dSAlex Tomas size = 32 * 1024; 3036c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3037c9de560dSAlex Tomas size = 64 * 1024; 3038c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3039c9de560dSAlex Tomas size = 128 * 1024; 3040c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3041c9de560dSAlex Tomas size = 256 * 1024; 3042c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3043c9de560dSAlex Tomas size = 512 * 1024; 3044c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3045c9de560dSAlex Tomas size = 1024 * 1024; 30461930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 3047c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 30481930479cSValerie Clement (21 - bsbits)) << 21; 30491930479cSValerie Clement size = 2 * 1024 * 1024; 30501930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 3051c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3052c9de560dSAlex Tomas (22 - bsbits)) << 22; 3053c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3054c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 30551930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 3056c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3057c9de560dSAlex Tomas (23 - bsbits)) << 23; 3058c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3059c9de560dSAlex Tomas } else { 3060c9de560dSAlex Tomas start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits; 3061c9de560dSAlex Tomas size = ac->ac_o_ex.fe_len << bsbits; 3062c9de560dSAlex Tomas } 30635a0790c2SAndi Kleen size = size >> bsbits; 30645a0790c2SAndi Kleen start = start_off >> bsbits; 3065c9de560dSAlex Tomas 3066c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 3067c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 3068c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 3069c9de560dSAlex Tomas start = ar->lleft + 1; 3070c9de560dSAlex Tomas } 3071c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 3072c9de560dSAlex Tomas size -= start + size - ar->lright; 3073c9de560dSAlex Tomas 3074c9de560dSAlex Tomas end = start + size; 3075c9de560dSAlex Tomas 3076c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 3077c9de560dSAlex Tomas rcu_read_lock(); 30789a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3079498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3080c9de560dSAlex Tomas 3081c9de560dSAlex Tomas if (pa->pa_deleted) 3082c9de560dSAlex Tomas continue; 3083c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3084c9de560dSAlex Tomas if (pa->pa_deleted) { 3085c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3086c9de560dSAlex Tomas continue; 3087c9de560dSAlex Tomas } 3088c9de560dSAlex Tomas 308953accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 309053accfa9STheodore Ts'o pa->pa_len); 3091c9de560dSAlex Tomas 3092c9de560dSAlex Tomas /* PA must not overlap original request */ 3093c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 3094c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 3095c9de560dSAlex Tomas 309638877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 309738877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 3098c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3099c9de560dSAlex Tomas continue; 3100c9de560dSAlex Tomas } 3101c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3102c9de560dSAlex Tomas 310338877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 3104c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3105c9de560dSAlex Tomas BUG_ON(pa_end < start); 3106c9de560dSAlex Tomas start = pa_end; 310738877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3108c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3109c9de560dSAlex Tomas end = pa->pa_lstart; 3110c9de560dSAlex Tomas } 3111c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3112c9de560dSAlex Tomas } 3113c9de560dSAlex Tomas rcu_read_unlock(); 3114c9de560dSAlex Tomas size = end - start; 3115c9de560dSAlex Tomas 3116c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3117c9de560dSAlex Tomas rcu_read_lock(); 31189a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3119498e5f24STheodore Ts'o ext4_lblk_t pa_end; 312053accfa9STheodore Ts'o 3121c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3122c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 312353accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 312453accfa9STheodore Ts'o pa->pa_len); 3125c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3126c9de560dSAlex Tomas } 3127c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3128c9de560dSAlex Tomas } 3129c9de560dSAlex Tomas rcu_read_unlock(); 3130c9de560dSAlex Tomas 3131c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3132c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 31339d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 31349d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 3135c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3136c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3137c9de560dSAlex Tomas } 3138c9de560dSAlex Tomas BUG_ON(start + size <= ac->ac_o_ex.fe_logical && 3139c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical); 31407137d7a4STheodore Ts'o BUG_ON(size <= 0 || size > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 3141c9de560dSAlex Tomas 3142c9de560dSAlex Tomas /* now prepare goal request */ 3143c9de560dSAlex Tomas 3144c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3145c9de560dSAlex Tomas * placement or satisfy big request as is */ 3146c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 314753accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 3148c9de560dSAlex Tomas 3149c9de560dSAlex Tomas /* define goal start in order to merge */ 3150c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3151c9de560dSAlex Tomas /* merge to the right */ 3152c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3153c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3154c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3155c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3156c9de560dSAlex Tomas } 3157c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3158c9de560dSAlex Tomas /* merge to the left */ 3159c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3160c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3161c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3162c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3163c9de560dSAlex Tomas } 3164c9de560dSAlex Tomas 31656ba495e9STheodore Ts'o mb_debug(1, "goal: %u(was %u) blocks at %u\n", (unsigned) size, 3166c9de560dSAlex Tomas (unsigned) orig_size, (unsigned) start); 3167c9de560dSAlex Tomas } 3168c9de560dSAlex Tomas 3169c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3170c9de560dSAlex Tomas { 3171c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3172c9de560dSAlex Tomas 3173c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3174c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3175c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3176291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 3177c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3178c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3179c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3180c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3181c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3182c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3183c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3184c9de560dSAlex Tomas } 3185c9de560dSAlex Tomas 3186296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 3187296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 3188296c355cSTheodore Ts'o else 3189296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 3190c9de560dSAlex Tomas } 3191c9de560dSAlex Tomas 3192c9de560dSAlex Tomas /* 3193b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 3194b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 3195b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 3196b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 3197b844167eSCurt Wohlgemuth */ 3198b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 3199b844167eSCurt Wohlgemuth { 3200b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 3201b844167eSCurt Wohlgemuth 3202400db9d3SZheng Liu if (pa && pa->pa_type == MB_INODE_PA) 3203400db9d3SZheng Liu pa->pa_free += ac->ac_b_ex.fe_len; 3204b844167eSCurt Wohlgemuth } 3205b844167eSCurt Wohlgemuth 3206b844167eSCurt Wohlgemuth /* 3207c9de560dSAlex Tomas * use blocks preallocated to inode 3208c9de560dSAlex Tomas */ 3209c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3210c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3211c9de560dSAlex Tomas { 321253accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3213c9de560dSAlex Tomas ext4_fsblk_t start; 3214c9de560dSAlex Tomas ext4_fsblk_t end; 3215c9de560dSAlex Tomas int len; 3216c9de560dSAlex Tomas 3217c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3218c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 321953accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 322053accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 322153accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 3222c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3223c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3224c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3225c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3226c9de560dSAlex Tomas ac->ac_pa = pa; 3227c9de560dSAlex Tomas 3228c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 322953accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 3230c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3231c9de560dSAlex Tomas pa->pa_free -= len; 3232c9de560dSAlex Tomas 32336ba495e9STheodore Ts'o mb_debug(1, "use %llu/%u from inode pa %p\n", start, len, pa); 3234c9de560dSAlex Tomas } 3235c9de560dSAlex Tomas 3236c9de560dSAlex Tomas /* 3237c9de560dSAlex Tomas * use blocks preallocated to locality group 3238c9de560dSAlex Tomas */ 3239c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3240c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3241c9de560dSAlex Tomas { 324203cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 32436be2ded1SAneesh Kumar K.V 3244c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3245c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3246c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3247c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3248c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3249c9de560dSAlex Tomas ac->ac_pa = pa; 3250c9de560dSAlex Tomas 3251c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 325226346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3253c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 325426346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 325526346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3256c9de560dSAlex Tomas */ 32576ba495e9STheodore Ts'o mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa); 3258c9de560dSAlex Tomas } 3259c9de560dSAlex Tomas 3260c9de560dSAlex Tomas /* 32615e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 32625e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 32635e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 32645e745b04SAneesh Kumar K.V * from the goal block. 32655e745b04SAneesh Kumar K.V */ 32665e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 32675e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 32685e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 32695e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 32705e745b04SAneesh Kumar K.V { 32715e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 32725e745b04SAneesh Kumar K.V 32735e745b04SAneesh Kumar K.V if (cpa == NULL) { 32745e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 32755e745b04SAneesh Kumar K.V return pa; 32765e745b04SAneesh Kumar K.V } 32775e745b04SAneesh Kumar K.V cur_distance = abs(goal_block - cpa->pa_pstart); 32785e745b04SAneesh Kumar K.V new_distance = abs(goal_block - pa->pa_pstart); 32795e745b04SAneesh Kumar K.V 32805a54b2f1SColy Li if (cur_distance <= new_distance) 32815e745b04SAneesh Kumar K.V return cpa; 32825e745b04SAneesh Kumar K.V 32835e745b04SAneesh Kumar K.V /* drop the previous reference */ 32845e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 32855e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 32865e745b04SAneesh Kumar K.V return pa; 32875e745b04SAneesh Kumar K.V } 32885e745b04SAneesh Kumar K.V 32895e745b04SAneesh Kumar K.V /* 3290c9de560dSAlex Tomas * search goal blocks in preallocated space 3291c9de560dSAlex Tomas */ 32924ddfef7bSEric Sandeen static noinline_for_stack int 32934ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3294c9de560dSAlex Tomas { 329553accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 32966be2ded1SAneesh Kumar K.V int order, i; 3297c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3298c9de560dSAlex Tomas struct ext4_locality_group *lg; 32995e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 33005e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 3301c9de560dSAlex Tomas 3302c9de560dSAlex Tomas /* only data can be preallocated */ 3303c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3304c9de560dSAlex Tomas return 0; 3305c9de560dSAlex Tomas 3306c9de560dSAlex Tomas /* first, try per-file preallocation */ 3307c9de560dSAlex Tomas rcu_read_lock(); 33089a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3309c9de560dSAlex Tomas 3310c9de560dSAlex Tomas /* all fields in this condition don't change, 3311c9de560dSAlex Tomas * so we can skip locking for them */ 3312c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 331353accfa9STheodore Ts'o ac->ac_o_ex.fe_logical >= (pa->pa_lstart + 331453accfa9STheodore Ts'o EXT4_C2B(sbi, pa->pa_len))) 3315c9de560dSAlex Tomas continue; 3316c9de560dSAlex Tomas 3317fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 331812e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 331953accfa9STheodore Ts'o (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > 332053accfa9STheodore Ts'o EXT4_MAX_BLOCK_FILE_PHYS)) 3321fb0a387dSEric Sandeen continue; 3322fb0a387dSEric Sandeen 3323c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3324c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3325c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3326c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3327c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3328c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3329c9de560dSAlex Tomas ac->ac_criteria = 10; 3330c9de560dSAlex Tomas rcu_read_unlock(); 3331c9de560dSAlex Tomas return 1; 3332c9de560dSAlex Tomas } 3333c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3334c9de560dSAlex Tomas } 3335c9de560dSAlex Tomas rcu_read_unlock(); 3336c9de560dSAlex Tomas 3337c9de560dSAlex Tomas /* can we use group allocation? */ 3338c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 3339c9de560dSAlex Tomas return 0; 3340c9de560dSAlex Tomas 3341c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3342c9de560dSAlex Tomas lg = ac->ac_lg; 3343c9de560dSAlex Tomas if (lg == NULL) 3344c9de560dSAlex Tomas return 0; 33456be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 33466be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 33476be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 33486be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 3349c9de560dSAlex Tomas 3350bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 33515e745b04SAneesh Kumar K.V /* 33525e745b04SAneesh Kumar K.V * search for the prealloc space that is having 33535e745b04SAneesh Kumar K.V * minimal distance from the goal block. 33545e745b04SAneesh Kumar K.V */ 33556be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 3356c9de560dSAlex Tomas rcu_read_lock(); 33576be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 33586be2ded1SAneesh Kumar K.V pa_inode_list) { 3359c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 33606be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 33616be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 33625e745b04SAneesh Kumar K.V 33635e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 33645e745b04SAneesh Kumar K.V pa, cpa); 33655e745b04SAneesh Kumar K.V } 3366c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 33675e745b04SAneesh Kumar K.V } 33685e745b04SAneesh Kumar K.V rcu_read_unlock(); 33695e745b04SAneesh Kumar K.V } 33705e745b04SAneesh Kumar K.V if (cpa) { 33715e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 3372c9de560dSAlex Tomas ac->ac_criteria = 20; 3373c9de560dSAlex Tomas return 1; 3374c9de560dSAlex Tomas } 3375c9de560dSAlex Tomas return 0; 3376c9de560dSAlex Tomas } 3377c9de560dSAlex Tomas 3378c9de560dSAlex Tomas /* 33797a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 33807a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 33817a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 3382955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 33837a2fcbf7SAneesh Kumar K.V */ 33847a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 33857a2fcbf7SAneesh Kumar K.V ext4_group_t group) 33867a2fcbf7SAneesh Kumar K.V { 33877a2fcbf7SAneesh Kumar K.V struct rb_node *n; 33887a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 33897a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 33907a2fcbf7SAneesh Kumar K.V 33917a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 33927a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 33937a2fcbf7SAneesh Kumar K.V 33947a2fcbf7SAneesh Kumar K.V while (n) { 339518aadd47SBobi Jam entry = rb_entry(n, struct ext4_free_data, efd_node); 339618aadd47SBobi Jam ext4_set_bits(bitmap, entry->efd_start_cluster, entry->efd_count); 33977a2fcbf7SAneesh Kumar K.V n = rb_next(n); 33987a2fcbf7SAneesh Kumar K.V } 33997a2fcbf7SAneesh Kumar K.V return; 34007a2fcbf7SAneesh Kumar K.V } 34017a2fcbf7SAneesh Kumar K.V 34027a2fcbf7SAneesh Kumar K.V /* 3403c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3404c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3405955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 3406c9de560dSAlex Tomas */ 3407089ceeccSEric Sandeen static noinline_for_stack 3408089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3409c9de560dSAlex Tomas ext4_group_t group) 3410c9de560dSAlex Tomas { 3411c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3412c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3413c9de560dSAlex Tomas struct list_head *cur; 3414c9de560dSAlex Tomas ext4_group_t groupnr; 3415c9de560dSAlex Tomas ext4_grpblk_t start; 3416c9de560dSAlex Tomas int preallocated = 0; 3417c9de560dSAlex Tomas int len; 3418c9de560dSAlex Tomas 3419c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3420c9de560dSAlex Tomas * so the only competing code is preallocation use. 3421c9de560dSAlex Tomas * we don't need any locking here 3422c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3423c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3424c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3425c9de560dSAlex Tomas * is dropping preallocation 3426c9de560dSAlex Tomas */ 3427c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3428c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3429c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3430c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3431c9de560dSAlex Tomas &groupnr, &start); 3432c9de560dSAlex Tomas len = pa->pa_len; 3433c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3434c9de560dSAlex Tomas if (unlikely(len == 0)) 3435c9de560dSAlex Tomas continue; 3436c9de560dSAlex Tomas BUG_ON(groupnr != group); 3437c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, start, len); 3438c9de560dSAlex Tomas preallocated += len; 3439c9de560dSAlex Tomas } 34406ba495e9STheodore Ts'o mb_debug(1, "prellocated %u for group %u\n", preallocated, group); 3441c9de560dSAlex Tomas } 3442c9de560dSAlex Tomas 3443c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3444c9de560dSAlex Tomas { 3445c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3446c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 34474e8d2139SJunho Ryu 34484e8d2139SJunho Ryu BUG_ON(atomic_read(&pa->pa_count)); 34494e8d2139SJunho Ryu BUG_ON(pa->pa_deleted == 0); 3450c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3451c9de560dSAlex Tomas } 3452c9de560dSAlex Tomas 3453c9de560dSAlex Tomas /* 3454c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3455c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3456c9de560dSAlex Tomas */ 3457c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3458c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3459c9de560dSAlex Tomas { 3460a9df9a49STheodore Ts'o ext4_group_t grp; 3461d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 3462c9de560dSAlex Tomas 3463c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3464c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 34654e8d2139SJunho Ryu if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) { 34664e8d2139SJunho Ryu spin_unlock(&pa->pa_lock); 34674e8d2139SJunho Ryu return; 34684e8d2139SJunho Ryu } 34694e8d2139SJunho Ryu 3470c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3471c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3472c9de560dSAlex Tomas return; 3473c9de560dSAlex Tomas } 3474c9de560dSAlex Tomas 3475c9de560dSAlex Tomas pa->pa_deleted = 1; 3476c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3477c9de560dSAlex Tomas 3478d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 3479cc0fb9adSAneesh Kumar K.V /* 3480cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 3481cc0fb9adSAneesh Kumar K.V * next group when pa is used up 3482cc0fb9adSAneesh Kumar K.V */ 3483cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3484d33a1976SEric Sandeen grp_blk--; 3485d33a1976SEric Sandeen 3486bd86298eSLukas Czerner grp = ext4_get_group_number(sb, grp_blk); 3487c9de560dSAlex Tomas 3488c9de560dSAlex Tomas /* 3489c9de560dSAlex Tomas * possible race: 3490c9de560dSAlex Tomas * 3491c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3492c9de560dSAlex Tomas * find block B in PA 3493c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3494c9de560dSAlex Tomas * mark B in on-disk bitmap 3495c9de560dSAlex Tomas * drop PA from group 3496c9de560dSAlex Tomas * mark all PAs in buddy 3497c9de560dSAlex Tomas * 3498c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3499c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3500c9de560dSAlex Tomas * against that pair 3501c9de560dSAlex Tomas */ 3502c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3503c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3504c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3505c9de560dSAlex Tomas 3506c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3507c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3508c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3509c9de560dSAlex Tomas 3510c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3511c9de560dSAlex Tomas } 3512c9de560dSAlex Tomas 3513c9de560dSAlex Tomas /* 3514c9de560dSAlex Tomas * creates new preallocated space for given inode 3515c9de560dSAlex Tomas */ 35164ddfef7bSEric Sandeen static noinline_for_stack int 35174ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3518c9de560dSAlex Tomas { 3519c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 352053accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 3521c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3522c9de560dSAlex Tomas struct ext4_group_info *grp; 3523c9de560dSAlex Tomas struct ext4_inode_info *ei; 3524c9de560dSAlex Tomas 3525c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3526c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3527c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3528c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3529c9de560dSAlex Tomas 3530c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3531c9de560dSAlex Tomas if (pa == NULL) 3532c9de560dSAlex Tomas return -ENOMEM; 3533c9de560dSAlex Tomas 3534c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3535c9de560dSAlex Tomas int winl; 3536c9de560dSAlex Tomas int wins; 3537c9de560dSAlex Tomas int win; 3538c9de560dSAlex Tomas int offs; 3539c9de560dSAlex Tomas 3540c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3541c9de560dSAlex Tomas * so, found space must get proper lstart 3542c9de560dSAlex Tomas * to cover original request */ 3543c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3544c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3545c9de560dSAlex Tomas 3546c9de560dSAlex Tomas /* we're limited by original request in that 3547c9de560dSAlex Tomas * logical block must be covered any way 3548c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3549c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3550c9de560dSAlex Tomas 3551c9de560dSAlex Tomas /* also, we should cover whole original request */ 355253accfa9STheodore Ts'o wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); 3553c9de560dSAlex Tomas 3554c9de560dSAlex Tomas /* the smallest one defines real window */ 3555c9de560dSAlex Tomas win = min(winl, wins); 3556c9de560dSAlex Tomas 355753accfa9STheodore Ts'o offs = ac->ac_o_ex.fe_logical % 355853accfa9STheodore Ts'o EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3559c9de560dSAlex Tomas if (offs && offs < win) 3560c9de560dSAlex Tomas win = offs; 3561c9de560dSAlex Tomas 356253accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - 3563810da240SLukas Czerner EXT4_NUM_B2C(sbi, win); 3564c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3565c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3566c9de560dSAlex Tomas } 3567c9de560dSAlex Tomas 3568c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3569c9de560dSAlex Tomas * allocated blocks for history */ 3570c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3571c9de560dSAlex Tomas 3572c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3573c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3574c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3575c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3576c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3577c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3578d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3579d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3580c9de560dSAlex Tomas pa->pa_deleted = 0; 3581cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 3582c9de560dSAlex Tomas 35836ba495e9STheodore Ts'o mb_debug(1, "new inode pa %p: %llu/%u for %u\n", pa, 3584c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 35859bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 3586c9de560dSAlex Tomas 3587c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 358853accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 3589c9de560dSAlex Tomas 3590c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3591c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3592c9de560dSAlex Tomas 3593c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3594c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3595c9de560dSAlex Tomas 3596c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3597c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3598c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3599c9de560dSAlex Tomas 3600c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3601c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3602c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3603c9de560dSAlex Tomas 3604c9de560dSAlex Tomas return 0; 3605c9de560dSAlex Tomas } 3606c9de560dSAlex Tomas 3607c9de560dSAlex Tomas /* 3608c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3609c9de560dSAlex Tomas */ 36104ddfef7bSEric Sandeen static noinline_for_stack int 36114ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3612c9de560dSAlex Tomas { 3613c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3614c9de560dSAlex Tomas struct ext4_locality_group *lg; 3615c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3616c9de560dSAlex Tomas struct ext4_group_info *grp; 3617c9de560dSAlex Tomas 3618c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3619c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3620c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3621c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3622c9de560dSAlex Tomas 3623c9de560dSAlex Tomas BUG_ON(ext4_pspace_cachep == NULL); 3624c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3625c9de560dSAlex Tomas if (pa == NULL) 3626c9de560dSAlex Tomas return -ENOMEM; 3627c9de560dSAlex Tomas 3628c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3629c9de560dSAlex Tomas * allocated blocks for history */ 3630c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3631c9de560dSAlex Tomas 3632c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3633c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 3634c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3635c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3636c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3637c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 36386be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3639d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3640c9de560dSAlex Tomas pa->pa_deleted = 0; 3641cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 3642c9de560dSAlex Tomas 36436ba495e9STheodore Ts'o mb_debug(1, "new group pa %p: %llu/%u for %u\n", pa, 3644c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 36459bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 3646c9de560dSAlex Tomas 3647c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3648c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3649c9de560dSAlex Tomas 3650c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3651c9de560dSAlex Tomas lg = ac->ac_lg; 3652c9de560dSAlex Tomas BUG_ON(lg == NULL); 3653c9de560dSAlex Tomas 3654c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 3655c9de560dSAlex Tomas pa->pa_inode = NULL; 3656c9de560dSAlex Tomas 3657c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3658c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3659c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3660c9de560dSAlex Tomas 36616be2ded1SAneesh Kumar K.V /* 36626be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 36636be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 36646be2ded1SAneesh Kumar K.V */ 3665c9de560dSAlex Tomas return 0; 3666c9de560dSAlex Tomas } 3667c9de560dSAlex Tomas 3668c9de560dSAlex Tomas static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 3669c9de560dSAlex Tomas { 3670c9de560dSAlex Tomas int err; 3671c9de560dSAlex Tomas 3672c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 3673c9de560dSAlex Tomas err = ext4_mb_new_group_pa(ac); 3674c9de560dSAlex Tomas else 3675c9de560dSAlex Tomas err = ext4_mb_new_inode_pa(ac); 3676c9de560dSAlex Tomas return err; 3677c9de560dSAlex Tomas } 3678c9de560dSAlex Tomas 3679c9de560dSAlex Tomas /* 3680c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 3681c9de560dSAlex Tomas * in-core bitmap and buddy. 3682c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 3683c9de560dSAlex Tomas * nobody else can find/use it. 3684c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 3685c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 3686c9de560dSAlex Tomas */ 36874ddfef7bSEric Sandeen static noinline_for_stack int 36884ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 36893e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3690c9de560dSAlex Tomas { 3691c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3692c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3693498e5f24STheodore Ts'o unsigned int end; 3694498e5f24STheodore Ts'o unsigned int next; 3695c9de560dSAlex Tomas ext4_group_t group; 3696c9de560dSAlex Tomas ext4_grpblk_t bit; 3697ba80b101STheodore Ts'o unsigned long long grp_blk_start; 3698c9de560dSAlex Tomas int err = 0; 3699c9de560dSAlex Tomas int free = 0; 3700c9de560dSAlex Tomas 3701c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3702c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 370353accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 3704c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3705c9de560dSAlex Tomas end = bit + pa->pa_len; 3706c9de560dSAlex Tomas 3707c9de560dSAlex Tomas while (bit < end) { 3708ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 3709c9de560dSAlex Tomas if (bit >= end) 3710c9de560dSAlex Tomas break; 3711ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 37126ba495e9STheodore Ts'o mb_debug(1, " free preallocated %u/%u in group %u\n", 37135a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 37145a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 3715c9de560dSAlex Tomas free += next - bit; 3716c9de560dSAlex Tomas 37173e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 371853accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 371953accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 3720a9c667f8SLukas Czerner next - bit); 3721c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 3722c9de560dSAlex Tomas bit = next + 1; 3723c9de560dSAlex Tomas } 3724c9de560dSAlex Tomas if (free != pa->pa_free) { 37259d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 37269d8b9ec4STheodore Ts'o "pa %p: logic %lu, phys. %lu, len %lu", 3727c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 3728c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 3729c9de560dSAlex Tomas (unsigned long) pa->pa_len); 3730e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 373126346ff6SAneesh Kumar K.V free, pa->pa_free); 3732e56eb659SAneesh Kumar K.V /* 3733e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 3734e56eb659SAneesh Kumar K.V * from the bitmap and continue. 3735e56eb659SAneesh Kumar K.V */ 3736c9de560dSAlex Tomas } 3737c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 3738c9de560dSAlex Tomas 3739c9de560dSAlex Tomas return err; 3740c9de560dSAlex Tomas } 3741c9de560dSAlex Tomas 37424ddfef7bSEric Sandeen static noinline_for_stack int 37434ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 37443e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3745c9de560dSAlex Tomas { 3746c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3747c9de560dSAlex Tomas ext4_group_t group; 3748c9de560dSAlex Tomas ext4_grpblk_t bit; 3749c9de560dSAlex Tomas 375060e07cf5SYongqiang Yang trace_ext4_mb_release_group_pa(sb, pa); 3751c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3752c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3753c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3754c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 3755c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 37563e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 3757c9de560dSAlex Tomas 3758c9de560dSAlex Tomas return 0; 3759c9de560dSAlex Tomas } 3760c9de560dSAlex Tomas 3761c9de560dSAlex Tomas /* 3762c9de560dSAlex Tomas * releases all preallocations in given group 3763c9de560dSAlex Tomas * 3764c9de560dSAlex Tomas * first, we need to decide discard policy: 3765c9de560dSAlex Tomas * - when do we discard 3766c9de560dSAlex Tomas * 1) ENOSPC 3767c9de560dSAlex Tomas * - how many do we discard 3768c9de560dSAlex Tomas * 1) how many requested 3769c9de560dSAlex Tomas */ 37704ddfef7bSEric Sandeen static noinline_for_stack int 37714ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 3772c9de560dSAlex Tomas ext4_group_t group, int needed) 3773c9de560dSAlex Tomas { 3774c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3775c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3776c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3777c9de560dSAlex Tomas struct list_head list; 3778c9de560dSAlex Tomas struct ext4_buddy e4b; 3779c9de560dSAlex Tomas int err; 3780c9de560dSAlex Tomas int busy = 0; 3781c9de560dSAlex Tomas int free = 0; 3782c9de560dSAlex Tomas 37836ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for group %u\n", group); 3784c9de560dSAlex Tomas 3785c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 3786c9de560dSAlex Tomas return 0; 3787c9de560dSAlex Tomas 3788574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 3789c9de560dSAlex Tomas if (bitmap_bh == NULL) { 379012062dddSEric Sandeen ext4_error(sb, "Error reading block bitmap for %u", group); 3791ce89f46cSAneesh Kumar K.V return 0; 3792c9de560dSAlex Tomas } 3793c9de560dSAlex Tomas 3794c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3795ce89f46cSAneesh Kumar K.V if (err) { 379612062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", group); 3797ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 3798ce89f46cSAneesh Kumar K.V return 0; 3799ce89f46cSAneesh Kumar K.V } 3800c9de560dSAlex Tomas 3801c9de560dSAlex Tomas if (needed == 0) 38027137d7a4STheodore Ts'o needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 3803c9de560dSAlex Tomas 3804c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3805c9de560dSAlex Tomas repeat: 3806c9de560dSAlex Tomas ext4_lock_group(sb, group); 3807c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 3808c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 3809c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3810c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3811c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3812c9de560dSAlex Tomas busy = 1; 3813c9de560dSAlex Tomas continue; 3814c9de560dSAlex Tomas } 3815c9de560dSAlex Tomas if (pa->pa_deleted) { 3816c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3817c9de560dSAlex Tomas continue; 3818c9de560dSAlex Tomas } 3819c9de560dSAlex Tomas 3820c9de560dSAlex Tomas /* seems this one can be freed ... */ 3821c9de560dSAlex Tomas pa->pa_deleted = 1; 3822c9de560dSAlex Tomas 3823c9de560dSAlex Tomas /* we can trust pa_free ... */ 3824c9de560dSAlex Tomas free += pa->pa_free; 3825c9de560dSAlex Tomas 3826c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3827c9de560dSAlex Tomas 3828c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3829c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3830c9de560dSAlex Tomas } 3831c9de560dSAlex Tomas 3832c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 3833c9de560dSAlex Tomas if (free < needed && busy) { 3834c9de560dSAlex Tomas busy = 0; 3835c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3836bb8b20edSLukas Czerner cond_resched(); 3837c9de560dSAlex Tomas goto repeat; 3838c9de560dSAlex Tomas } 3839c9de560dSAlex Tomas 3840c9de560dSAlex Tomas /* found anything to free? */ 3841c9de560dSAlex Tomas if (list_empty(&list)) { 3842c9de560dSAlex Tomas BUG_ON(free != 0); 3843c9de560dSAlex Tomas goto out; 3844c9de560dSAlex Tomas } 3845c9de560dSAlex Tomas 3846c9de560dSAlex Tomas /* now free all selected PAs */ 3847c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3848c9de560dSAlex Tomas 3849c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 3850c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3851c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3852c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3853c9de560dSAlex Tomas 3854cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 38553e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 3856c9de560dSAlex Tomas else 38573e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 3858c9de560dSAlex Tomas 3859c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3860c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3861c9de560dSAlex Tomas } 3862c9de560dSAlex Tomas 3863c9de560dSAlex Tomas out: 3864c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3865e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3866c9de560dSAlex Tomas put_bh(bitmap_bh); 3867c9de560dSAlex Tomas return free; 3868c9de560dSAlex Tomas } 3869c9de560dSAlex Tomas 3870c9de560dSAlex Tomas /* 3871c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 3872c9de560dSAlex Tomas * 3873c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 3874c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 3875c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 3876c9de560dSAlex Tomas * 3877c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 3878c9de560dSAlex Tomas */ 3879c2ea3fdeSTheodore Ts'o void ext4_discard_preallocations(struct inode *inode) 3880c9de560dSAlex Tomas { 3881c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 3882c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 3883c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3884c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3885c9de560dSAlex Tomas ext4_group_t group = 0; 3886c9de560dSAlex Tomas struct list_head list; 3887c9de560dSAlex Tomas struct ext4_buddy e4b; 3888c9de560dSAlex Tomas int err; 3889c9de560dSAlex Tomas 3890c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 3891c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 3892c9de560dSAlex Tomas return; 3893c9de560dSAlex Tomas } 3894c9de560dSAlex Tomas 38956ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino); 38969bffad1eSTheodore Ts'o trace_ext4_discard_preallocations(inode); 3897c9de560dSAlex Tomas 3898c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3899c9de560dSAlex Tomas 3900c9de560dSAlex Tomas repeat: 3901c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 3902c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 3903c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 3904c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 3905c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 3906c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 3907c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3908c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3909c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 3910c9de560dSAlex Tomas * use preallocation while we're discarding it */ 3911c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3912c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 39139d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 39149d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 3915c9de560dSAlex Tomas WARN_ON(1); 3916c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 3917c9de560dSAlex Tomas goto repeat; 3918c9de560dSAlex Tomas 3919c9de560dSAlex Tomas } 3920c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3921c9de560dSAlex Tomas pa->pa_deleted = 1; 3922c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3923c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3924c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3925c9de560dSAlex Tomas continue; 3926c9de560dSAlex Tomas } 3927c9de560dSAlex Tomas 3928c9de560dSAlex Tomas /* someone is deleting pa right now */ 3929c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3930c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3931c9de560dSAlex Tomas 3932c9de560dSAlex Tomas /* we have to wait here because pa_deleted 3933c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 3934c9de560dSAlex Tomas * the list. as we might be called from 3935c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 3936c9de560dSAlex Tomas * and concurrent thread which is unlinking 3937c9de560dSAlex Tomas * pa from inode's list may access already 3938c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 3939c9de560dSAlex Tomas 3940c9de560dSAlex Tomas /* XXX: if this happens too often, we can 3941c9de560dSAlex Tomas * add a flag to force wait only in case 3942c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 3943c9de560dSAlex Tomas * regular truncate */ 3944c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 3945c9de560dSAlex Tomas goto repeat; 3946c9de560dSAlex Tomas } 3947c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3948c9de560dSAlex Tomas 3949c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3950cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 3951bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 3952c9de560dSAlex Tomas 3953c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3954ce89f46cSAneesh Kumar K.V if (err) { 395512062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", 395612062dddSEric Sandeen group); 3957ce89f46cSAneesh Kumar K.V continue; 3958ce89f46cSAneesh Kumar K.V } 3959c9de560dSAlex Tomas 3960574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 3961c9de560dSAlex Tomas if (bitmap_bh == NULL) { 396212062dddSEric Sandeen ext4_error(sb, "Error reading block bitmap for %u", 396312062dddSEric Sandeen group); 3964e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3965ce89f46cSAneesh Kumar K.V continue; 3966c9de560dSAlex Tomas } 3967c9de560dSAlex Tomas 3968c9de560dSAlex Tomas ext4_lock_group(sb, group); 3969c9de560dSAlex Tomas list_del(&pa->pa_group_list); 39703e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 3971c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3972c9de560dSAlex Tomas 3973e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3974c9de560dSAlex Tomas put_bh(bitmap_bh); 3975c9de560dSAlex Tomas 3976c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3977c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3978c9de560dSAlex Tomas } 3979c9de560dSAlex Tomas } 3980c9de560dSAlex Tomas 39816ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 3982c9de560dSAlex Tomas static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 3983c9de560dSAlex Tomas { 3984c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 39858df9675fSTheodore Ts'o ext4_group_t ngroups, i; 3986c9de560dSAlex Tomas 3987a0b30c12STheodore Ts'o if (!ext4_mballoc_debug || 39884dd89fc6STheodore Ts'o (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) 3989e3570639SEric Sandeen return; 3990e3570639SEric Sandeen 39917f6a11e7SJoe Perches ext4_msg(ac->ac_sb, KERN_ERR, "Can't allocate:" 39929d8b9ec4STheodore Ts'o " Allocation context details:"); 39937f6a11e7SJoe Perches ext4_msg(ac->ac_sb, KERN_ERR, "status %d flags %d", 3994c9de560dSAlex Tomas ac->ac_status, ac->ac_flags); 39957f6a11e7SJoe Perches ext4_msg(ac->ac_sb, KERN_ERR, "orig %lu/%lu/%lu@%lu, " 39969d8b9ec4STheodore Ts'o "goal %lu/%lu/%lu@%lu, " 39979d8b9ec4STheodore Ts'o "best %lu/%lu/%lu@%lu cr %d", 3998c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_group, 3999c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_start, 4000c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_len, 4001c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_logical, 4002c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_group, 4003c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_start, 4004c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_len, 4005c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_logical, 4006c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_group, 4007c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_start, 4008c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_len, 4009c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_logical, 4010c9de560dSAlex Tomas (int)ac->ac_criteria); 40117f6a11e7SJoe Perches ext4_msg(ac->ac_sb, KERN_ERR, "%lu scanned, %d found", 40129d8b9ec4STheodore Ts'o ac->ac_ex_scanned, ac->ac_found); 40137f6a11e7SJoe Perches ext4_msg(ac->ac_sb, KERN_ERR, "groups: "); 40148df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 40158df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 4016c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 4017c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4018c9de560dSAlex Tomas ext4_grpblk_t start; 4019c9de560dSAlex Tomas struct list_head *cur; 4020c9de560dSAlex Tomas ext4_lock_group(sb, i); 4021c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4022c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 4023c9de560dSAlex Tomas pa_group_list); 4024c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4025c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4026c9de560dSAlex Tomas NULL, &start); 4027c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 40281c718505SAkira Fujita printk(KERN_ERR "PA:%u:%d:%u \n", i, 4029c9de560dSAlex Tomas start, pa->pa_len); 4030c9de560dSAlex Tomas } 403160bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 4032c9de560dSAlex Tomas 4033c9de560dSAlex Tomas if (grp->bb_free == 0) 4034c9de560dSAlex Tomas continue; 40351c718505SAkira Fujita printk(KERN_ERR "%u: %d/%d \n", 4036c9de560dSAlex Tomas i, grp->bb_free, grp->bb_fragments); 4037c9de560dSAlex Tomas } 4038c9de560dSAlex Tomas printk(KERN_ERR "\n"); 4039c9de560dSAlex Tomas } 4040c9de560dSAlex Tomas #else 4041c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4042c9de560dSAlex Tomas { 4043c9de560dSAlex Tomas return; 4044c9de560dSAlex Tomas } 4045c9de560dSAlex Tomas #endif 4046c9de560dSAlex Tomas 4047c9de560dSAlex Tomas /* 4048c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 4049c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 4050c9de560dSAlex Tomas * allocation which ever is larger 4051c9de560dSAlex Tomas * 4052b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 4053c9de560dSAlex Tomas */ 4054c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 4055c9de560dSAlex Tomas { 4056c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4057c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 4058c9de560dSAlex Tomas loff_t size, isize; 4059c9de560dSAlex Tomas 4060c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4061c9de560dSAlex Tomas return; 4062c9de560dSAlex Tomas 40634ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 40644ba74d00STheodore Ts'o return; 40654ba74d00STheodore Ts'o 406653accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 406750797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 406850797481STheodore Ts'o >> bsbits; 4069c9de560dSAlex Tomas 407050797481STheodore Ts'o if ((size == isize) && 407150797481STheodore Ts'o !ext4_fs_is_busy(sbi) && 407250797481STheodore Ts'o (atomic_read(&ac->ac_inode->i_writecount) == 0)) { 407350797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 407450797481STheodore Ts'o return; 407550797481STheodore Ts'o } 407650797481STheodore Ts'o 4077ebbe0277SRobin Dong if (sbi->s_mb_group_prealloc <= 0) { 4078ebbe0277SRobin Dong ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4079ebbe0277SRobin Dong return; 4080ebbe0277SRobin Dong } 4081ebbe0277SRobin Dong 4082c9de560dSAlex Tomas /* don't use group allocation for large files */ 408371780577STheodore Ts'o size = max(size, isize); 4084cc483f10STao Ma if (size > sbi->s_mb_stream_request) { 40854ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4086c9de560dSAlex Tomas return; 40874ba74d00STheodore Ts'o } 4088c9de560dSAlex Tomas 4089c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 4090c9de560dSAlex Tomas /* 4091c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 4092c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 4093c9de560dSAlex Tomas * request from multiple CPUs. 4094c9de560dSAlex Tomas */ 4095ca0c9584SChristoph Lameter ac->ac_lg = __this_cpu_ptr(sbi->s_locality_groups); 4096c9de560dSAlex Tomas 4097c9de560dSAlex Tomas /* we're going to use group allocation */ 4098c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4099c9de560dSAlex Tomas 4100c9de560dSAlex Tomas /* serialize all allocations in the group */ 4101c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4102c9de560dSAlex Tomas } 4103c9de560dSAlex Tomas 41044ddfef7bSEric Sandeen static noinline_for_stack int 41054ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4106c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4107c9de560dSAlex Tomas { 4108c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4109c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4110c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4111c9de560dSAlex Tomas ext4_group_t group; 4112498e5f24STheodore Ts'o unsigned int len; 4113498e5f24STheodore Ts'o ext4_fsblk_t goal; 4114c9de560dSAlex Tomas ext4_grpblk_t block; 4115c9de560dSAlex Tomas 4116c9de560dSAlex Tomas /* we can't allocate > group size */ 4117c9de560dSAlex Tomas len = ar->len; 4118c9de560dSAlex Tomas 4119c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 412040ae3487STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb)) 412140ae3487STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb); 4122c9de560dSAlex Tomas 4123c9de560dSAlex Tomas /* start searching from the goal */ 4124c9de560dSAlex Tomas goal = ar->goal; 4125c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4126c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4127c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4128c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4129c9de560dSAlex Tomas 4130c9de560dSAlex Tomas /* set up allocation goals */ 4131f5a44db5STheodore Ts'o ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical); 4132c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4133c9de560dSAlex Tomas ac->ac_sb = sb; 4134c9de560dSAlex Tomas ac->ac_inode = ar->inode; 413553accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 4136c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4137c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4138c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 413953accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 4140c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4141c9de560dSAlex Tomas 4142c9de560dSAlex Tomas /* we have to define context: we'll we work with a file or 4143c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4144c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4145c9de560dSAlex Tomas 41466ba495e9STheodore Ts'o mb_debug(1, "init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, " 4147c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4148c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4149c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4150c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4151c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 4152c9de560dSAlex Tomas atomic_read(&ar->inode->i_writecount) ? "" : "non-"); 4153c9de560dSAlex Tomas return 0; 4154c9de560dSAlex Tomas 4155c9de560dSAlex Tomas } 4156c9de560dSAlex Tomas 41576be2ded1SAneesh Kumar K.V static noinline_for_stack void 41586be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 41596be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 41606be2ded1SAneesh Kumar K.V int order, int total_entries) 41616be2ded1SAneesh Kumar K.V { 41626be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 41636be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 41646be2ded1SAneesh Kumar K.V struct list_head discard_list; 41656be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 41666be2ded1SAneesh Kumar K.V 41676ba495e9STheodore Ts'o mb_debug(1, "discard locality group preallocation\n"); 41686be2ded1SAneesh Kumar K.V 41696be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 41706be2ded1SAneesh Kumar K.V 41716be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 41726be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 41736be2ded1SAneesh Kumar K.V pa_inode_list) { 41746be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 41756be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 41766be2ded1SAneesh Kumar K.V /* 41776be2ded1SAneesh Kumar K.V * This is the pa that we just used 41786be2ded1SAneesh Kumar K.V * for block allocation. So don't 41796be2ded1SAneesh Kumar K.V * free that 41806be2ded1SAneesh Kumar K.V */ 41816be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 41826be2ded1SAneesh Kumar K.V continue; 41836be2ded1SAneesh Kumar K.V } 41846be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 41856be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 41866be2ded1SAneesh Kumar K.V continue; 41876be2ded1SAneesh Kumar K.V } 41886be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 4189cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 41906be2ded1SAneesh Kumar K.V 41916be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 41926be2ded1SAneesh Kumar K.V pa->pa_deleted = 1; 41936be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 41946be2ded1SAneesh Kumar K.V 41956be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 41966be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 41976be2ded1SAneesh Kumar K.V 41986be2ded1SAneesh Kumar K.V total_entries--; 41996be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 42006be2ded1SAneesh Kumar K.V /* 42016be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 42026be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 42036be2ded1SAneesh Kumar K.V * mak sure we don't call discard 42046be2ded1SAneesh Kumar K.V * soon for this list. 42056be2ded1SAneesh Kumar K.V */ 42066be2ded1SAneesh Kumar K.V break; 42076be2ded1SAneesh Kumar K.V } 42086be2ded1SAneesh Kumar K.V } 42096be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 42106be2ded1SAneesh Kumar K.V 42116be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 42126be2ded1SAneesh Kumar K.V 4213bd86298eSLukas Czerner group = ext4_get_group_number(sb, pa->pa_pstart); 42146be2ded1SAneesh Kumar K.V if (ext4_mb_load_buddy(sb, group, &e4b)) { 421512062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", 421612062dddSEric Sandeen group); 42176be2ded1SAneesh Kumar K.V continue; 42186be2ded1SAneesh Kumar K.V } 42196be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 42206be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 42213e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 42226be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 42236be2ded1SAneesh Kumar K.V 4224e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 42256be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 42266be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 42276be2ded1SAneesh Kumar K.V } 42286be2ded1SAneesh Kumar K.V } 42296be2ded1SAneesh Kumar K.V 42306be2ded1SAneesh Kumar K.V /* 42316be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 42326be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 42336be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 42346be2ded1SAneesh Kumar K.V * 42356be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 42366be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 42376be2ded1SAneesh Kumar K.V */ 42386be2ded1SAneesh Kumar K.V 42396be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 42406be2ded1SAneesh Kumar K.V { 42416be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 42426be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 42436be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 42446be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 42456be2ded1SAneesh Kumar K.V 42466be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 42476be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 42486be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 42496be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 42506be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 4251f1167009SNiu Yawei spin_lock(&lg->lg_prealloc_lock); 42526be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 42536be2ded1SAneesh Kumar K.V pa_inode_list) { 42546be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 42556be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 4256e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 42576be2ded1SAneesh Kumar K.V continue; 42586be2ded1SAneesh Kumar K.V } 42596be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 42606be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 42616be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 42626be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 42636be2ded1SAneesh Kumar K.V added = 1; 42646be2ded1SAneesh Kumar K.V /* 42656be2ded1SAneesh Kumar K.V * we want to count the total 42666be2ded1SAneesh Kumar K.V * number of entries in the list 42676be2ded1SAneesh Kumar K.V */ 42686be2ded1SAneesh Kumar K.V } 42696be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 42706be2ded1SAneesh Kumar K.V lg_prealloc_count++; 42716be2ded1SAneesh Kumar K.V } 42726be2ded1SAneesh Kumar K.V if (!added) 42736be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 42746be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 4275f1167009SNiu Yawei spin_unlock(&lg->lg_prealloc_lock); 42766be2ded1SAneesh Kumar K.V 42776be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 42786be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 42796be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 42806be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 42816be2ded1SAneesh Kumar K.V return; 42826be2ded1SAneesh Kumar K.V } 42836be2ded1SAneesh Kumar K.V return ; 42846be2ded1SAneesh Kumar K.V } 42856be2ded1SAneesh Kumar K.V 4286c9de560dSAlex Tomas /* 4287c9de560dSAlex Tomas * release all resource we used in allocation 4288c9de560dSAlex Tomas */ 4289c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4290c9de560dSAlex Tomas { 429153accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 42926be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 42936be2ded1SAneesh Kumar K.V if (pa) { 4294cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 4295c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 42966be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 429753accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 429853accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 42996be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 43006be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 43016be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 4302ba443916SAneesh Kumar K.V } 4303ba443916SAneesh Kumar K.V } 4304ba443916SAneesh Kumar K.V if (pa) { 43056be2ded1SAneesh Kumar K.V /* 43066be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 43076be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 43086be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 430944183d42SAmir Goldstein * doesn't grow big. 43106be2ded1SAneesh Kumar K.V */ 4311cc0fb9adSAneesh Kumar K.V if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) { 43126be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 43136be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 43146be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 43156be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 4316c9de560dSAlex Tomas } 43176be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 4318c9de560dSAlex Tomas } 4319c9de560dSAlex Tomas if (ac->ac_bitmap_page) 4320c9de560dSAlex Tomas page_cache_release(ac->ac_bitmap_page); 4321c9de560dSAlex Tomas if (ac->ac_buddy_page) 4322c9de560dSAlex Tomas page_cache_release(ac->ac_buddy_page); 4323c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4324c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4325c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4326c9de560dSAlex Tomas return 0; 4327c9de560dSAlex Tomas } 4328c9de560dSAlex Tomas 4329c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4330c9de560dSAlex Tomas { 43318df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 4332c9de560dSAlex Tomas int ret; 4333c9de560dSAlex Tomas int freed = 0; 4334c9de560dSAlex Tomas 43359bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 43368df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 4337c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4338c9de560dSAlex Tomas freed += ret; 4339c9de560dSAlex Tomas needed -= ret; 4340c9de560dSAlex Tomas } 4341c9de560dSAlex Tomas 4342c9de560dSAlex Tomas return freed; 4343c9de560dSAlex Tomas } 4344c9de560dSAlex Tomas 4345c9de560dSAlex Tomas /* 4346c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4347c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4348c9de560dSAlex Tomas * to usual allocation 4349c9de560dSAlex Tomas */ 4350c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4351c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4352c9de560dSAlex Tomas { 43536bc6e63fSAneesh Kumar K.V int freed; 4354256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4355c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4356c9de560dSAlex Tomas struct super_block *sb; 4357c9de560dSAlex Tomas ext4_fsblk_t block = 0; 435860e58e0fSMingming Cao unsigned int inquota = 0; 435953accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 4360c9de560dSAlex Tomas 4361b10a44c3STheodore Ts'o might_sleep(); 4362c9de560dSAlex Tomas sb = ar->inode->i_sb; 4363c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4364c9de560dSAlex Tomas 43659bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 4366ba80b101STheodore Ts'o 436745dc63e7SDmitry Monakhov /* Allow to use superuser reservation for quota file */ 436845dc63e7SDmitry Monakhov if (IS_NOQUOTA(ar->inode)) 436945dc63e7SDmitry Monakhov ar->flags |= EXT4_MB_USE_ROOT_BLOCKS; 437045dc63e7SDmitry Monakhov 4371d2a17637SMingming Cao /* 437260e58e0fSMingming Cao * For delayed allocation, we could skip the ENOSPC and 437360e58e0fSMingming Cao * EDQUOT check, as blocks and quotas have been already 437460e58e0fSMingming Cao * reserved when data being copied into pagecache. 437560e58e0fSMingming Cao */ 4376f2321097STheodore Ts'o if (ext4_test_inode_state(ar->inode, EXT4_STATE_DELALLOC_RESERVED)) 437760e58e0fSMingming Cao ar->flags |= EXT4_MB_DELALLOC_RESERVED; 437860e58e0fSMingming Cao else { 437960e58e0fSMingming Cao /* Without delayed allocation we need to verify 438060e58e0fSMingming Cao * there is enough free blocks to do block allocation 438160e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 4382d2a17637SMingming Cao */ 438355f020dbSAllison Henderson while (ar->len && 4384e7d5f315STheodore Ts'o ext4_claim_free_clusters(sbi, ar->len, ar->flags)) { 438555f020dbSAllison Henderson 4386030ba6bcSAneesh Kumar K.V /* let others to free the space */ 4387bb8b20edSLukas Czerner cond_resched(); 4388030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 4389030ba6bcSAneesh Kumar K.V } 4390030ba6bcSAneesh Kumar K.V if (!ar->len) { 439107031431SMingming Cao *errp = -ENOSPC; 439207031431SMingming Cao return 0; 439307031431SMingming Cao } 439453accfa9STheodore Ts'o reserv_clstrs = ar->len; 439555f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 439653accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 439753accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 439855f020dbSAllison Henderson } else { 439955f020dbSAllison Henderson while (ar->len && 440053accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 440153accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 440255f020dbSAllison Henderson 4403c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4404c9de560dSAlex Tomas ar->len--; 4405c9de560dSAlex Tomas } 440655f020dbSAllison Henderson } 440760e58e0fSMingming Cao inquota = ar->len; 4408c9de560dSAlex Tomas if (ar->len == 0) { 4409c9de560dSAlex Tomas *errp = -EDQUOT; 44106c7a120aSAditya Kali goto out; 4411c9de560dSAlex Tomas } 441260e58e0fSMingming Cao } 4413d2a17637SMingming Cao 441485556c9aSWei Yongjun ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS); 4415833576b3STheodore Ts'o if (!ac) { 4416363d4251SShen Feng ar->len = 0; 4417256bdb49SEric Sandeen *errp = -ENOMEM; 44186c7a120aSAditya Kali goto out; 4419256bdb49SEric Sandeen } 4420256bdb49SEric Sandeen 4421256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4422c9de560dSAlex Tomas if (*errp) { 4423c9de560dSAlex Tomas ar->len = 0; 44246c7a120aSAditya Kali goto out; 4425c9de560dSAlex Tomas } 4426c9de560dSAlex Tomas 4427256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 4428256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4429256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4430256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 4431c9de560dSAlex Tomas repeat: 4432c9de560dSAlex Tomas /* allocate space in core */ 44336c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 44342c00ef3eSAlexey Khoroshilov if (*errp) 44352c00ef3eSAlexey Khoroshilov goto discard_and_exit; 4436c9de560dSAlex Tomas 4437c9de560dSAlex Tomas /* as we've just preallocated more space than 44382c00ef3eSAlexey Khoroshilov * user requested originally, we store allocated 4439c9de560dSAlex Tomas * space in a special descriptor */ 4440256bdb49SEric Sandeen if (ac->ac_status == AC_STATUS_FOUND && 4441256bdb49SEric Sandeen ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 44422c00ef3eSAlexey Khoroshilov *errp = ext4_mb_new_preallocation(ac); 44432c00ef3eSAlexey Khoroshilov if (*errp) { 44442c00ef3eSAlexey Khoroshilov discard_and_exit: 44452c00ef3eSAlexey Khoroshilov ext4_discard_allocated_blocks(ac); 44462c00ef3eSAlexey Khoroshilov goto errout; 44472c00ef3eSAlexey Khoroshilov } 4448c9de560dSAlex Tomas } 4449256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 445053accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 4451519deca0SAneesh Kumar K.V if (*errp == -EAGAIN) { 44528556e8f3SAneesh Kumar K.V /* 44538556e8f3SAneesh Kumar K.V * drop the reference that we took 44548556e8f3SAneesh Kumar K.V * in ext4_mb_use_best_found 44558556e8f3SAneesh Kumar K.V */ 44568556e8f3SAneesh Kumar K.V ext4_mb_release_context(ac); 4457519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_group = 0; 4458519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_start = 0; 4459519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len = 0; 4460519deca0SAneesh Kumar K.V ac->ac_status = AC_STATUS_CONTINUE; 4461519deca0SAneesh Kumar K.V goto repeat; 44626d138cedSEric Sandeen } else if (*errp) { 4463b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 44646d138cedSEric Sandeen goto errout; 44656d138cedSEric Sandeen } else { 4466256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4467256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4468519deca0SAneesh Kumar K.V } 4469c9de560dSAlex Tomas } else { 4470256bdb49SEric Sandeen freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 4471c9de560dSAlex Tomas if (freed) 4472c9de560dSAlex Tomas goto repeat; 4473c9de560dSAlex Tomas *errp = -ENOSPC; 44746c7a120aSAditya Kali } 44756c7a120aSAditya Kali 44766d138cedSEric Sandeen errout: 44776c7a120aSAditya Kali if (*errp) { 4478256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4479c9de560dSAlex Tomas ar->len = 0; 4480256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4481c9de560dSAlex Tomas } 4482256bdb49SEric Sandeen ext4_mb_release_context(ac); 44836c7a120aSAditya Kali out: 44846c7a120aSAditya Kali if (ac) 4485363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 448660e58e0fSMingming Cao if (inquota && ar->len < inquota) 448753accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 44880087d9fbSAneesh Kumar K.V if (!ar->len) { 4489f2321097STheodore Ts'o if (!ext4_test_inode_state(ar->inode, 4490f2321097STheodore Ts'o EXT4_STATE_DELALLOC_RESERVED)) 44910087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 449257042651STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyclusters_counter, 449353accfa9STheodore Ts'o reserv_clstrs); 44940087d9fbSAneesh Kumar K.V } 4495c9de560dSAlex Tomas 44969bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 4497ba80b101STheodore Ts'o 4498c9de560dSAlex Tomas return block; 4499c9de560dSAlex Tomas } 4500c9de560dSAlex Tomas 4501c894058dSAneesh Kumar K.V /* 4502c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 4503c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 4504c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 4505c894058dSAneesh Kumar K.V */ 4506c894058dSAneesh Kumar K.V static int can_merge(struct ext4_free_data *entry1, 4507c894058dSAneesh Kumar K.V struct ext4_free_data *entry2) 4508c894058dSAneesh Kumar K.V { 450918aadd47SBobi Jam if ((entry1->efd_tid == entry2->efd_tid) && 451018aadd47SBobi Jam (entry1->efd_group == entry2->efd_group) && 451118aadd47SBobi Jam ((entry1->efd_start_cluster + entry1->efd_count) == entry2->efd_start_cluster)) 4512c894058dSAneesh Kumar K.V return 1; 4513c894058dSAneesh Kumar K.V return 0; 4514c894058dSAneesh Kumar K.V } 4515c894058dSAneesh Kumar K.V 45164ddfef7bSEric Sandeen static noinline_for_stack int 45174ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 45187a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 4519c9de560dSAlex Tomas { 4520e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 452184130193STheodore Ts'o ext4_grpblk_t cluster; 45227a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 4523c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4524c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4525c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4526c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 4527c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 4528c894058dSAneesh Kumar K.V 45290390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 4530c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4531c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4532c9de560dSAlex Tomas 453318aadd47SBobi Jam new_node = &new_entry->efd_node; 453418aadd47SBobi Jam cluster = new_entry->efd_start_cluster; 4535c9de560dSAlex Tomas 4536c894058dSAneesh Kumar K.V if (!*n) { 4537c894058dSAneesh Kumar K.V /* first free block exent. We need to 4538c894058dSAneesh Kumar K.V protect buddy cache from being freed, 4539c9de560dSAlex Tomas * otherwise we'll refresh it from 4540c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4541c9de560dSAlex Tomas * blocks */ 4542c9de560dSAlex Tomas page_cache_get(e4b->bd_buddy_page); 4543c9de560dSAlex Tomas page_cache_get(e4b->bd_bitmap_page); 4544c894058dSAneesh Kumar K.V } 4545c894058dSAneesh Kumar K.V while (*n) { 4546c894058dSAneesh Kumar K.V parent = *n; 454718aadd47SBobi Jam entry = rb_entry(parent, struct ext4_free_data, efd_node); 454818aadd47SBobi Jam if (cluster < entry->efd_start_cluster) 4549c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 455018aadd47SBobi Jam else if (cluster >= (entry->efd_start_cluster + entry->efd_count)) 4551c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 4552c894058dSAneesh Kumar K.V else { 4553e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 455484130193STheodore Ts'o ext4_group_first_block_no(sb, group) + 455584130193STheodore Ts'o EXT4_C2B(sbi, cluster), 4556e29136f8STheodore Ts'o "Block already on to-be-freed list"); 4557c894058dSAneesh Kumar K.V return 0; 4558c9de560dSAlex Tomas } 4559c9de560dSAlex Tomas } 4560c9de560dSAlex Tomas 4561c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 4562c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 4563c894058dSAneesh Kumar K.V 4564c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 4565c894058dSAneesh Kumar K.V node = rb_prev(new_node); 4566c894058dSAneesh Kumar K.V if (node) { 456718aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 45685d3ee208SDmitry Monakhov if (can_merge(entry, new_entry) && 45695d3ee208SDmitry Monakhov ext4_journal_callback_try_del(handle, &entry->efd_jce)) { 457018aadd47SBobi Jam new_entry->efd_start_cluster = entry->efd_start_cluster; 457118aadd47SBobi Jam new_entry->efd_count += entry->efd_count; 4572c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 457318aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 4574c9de560dSAlex Tomas } 4575c9de560dSAlex Tomas } 4576c894058dSAneesh Kumar K.V 4577c894058dSAneesh Kumar K.V node = rb_next(new_node); 4578c894058dSAneesh Kumar K.V if (node) { 457918aadd47SBobi Jam entry = rb_entry(node, struct ext4_free_data, efd_node); 45805d3ee208SDmitry Monakhov if (can_merge(new_entry, entry) && 45815d3ee208SDmitry Monakhov ext4_journal_callback_try_del(handle, &entry->efd_jce)) { 458218aadd47SBobi Jam new_entry->efd_count += entry->efd_count; 4583c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 458418aadd47SBobi Jam kmem_cache_free(ext4_free_data_cachep, entry); 4585c894058dSAneesh Kumar K.V } 4586c894058dSAneesh Kumar K.V } 45873e624fc7STheodore Ts'o /* Add the extent to transaction's private list */ 458818aadd47SBobi Jam ext4_journal_callback_add(handle, ext4_free_data_callback, 458918aadd47SBobi Jam &new_entry->efd_jce); 4590c9de560dSAlex Tomas return 0; 4591c9de560dSAlex Tomas } 4592c9de560dSAlex Tomas 459344338711STheodore Ts'o /** 459444338711STheodore Ts'o * ext4_free_blocks() -- Free given blocks and update quota 459544338711STheodore Ts'o * @handle: handle for this transaction 459644338711STheodore Ts'o * @inode: inode 459744338711STheodore Ts'o * @block: start physical block to free 459844338711STheodore Ts'o * @count: number of blocks to count 45995def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 4600c9de560dSAlex Tomas */ 460144338711STheodore Ts'o void ext4_free_blocks(handle_t *handle, struct inode *inode, 4602e6362609STheodore Ts'o struct buffer_head *bh, ext4_fsblk_t block, 4603e6362609STheodore Ts'o unsigned long count, int flags) 4604c9de560dSAlex Tomas { 460526346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 4606c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4607c9de560dSAlex Tomas struct ext4_group_desc *gdp; 4608498e5f24STheodore Ts'o unsigned int overflow; 4609c9de560dSAlex Tomas ext4_grpblk_t bit; 4610c9de560dSAlex Tomas struct buffer_head *gd_bh; 4611c9de560dSAlex Tomas ext4_group_t block_group; 4612c9de560dSAlex Tomas struct ext4_sb_info *sbi; 46137d734532SJan Kara struct ext4_inode_info *ei = EXT4_I(inode); 4614c9de560dSAlex Tomas struct ext4_buddy e4b; 461584130193STheodore Ts'o unsigned int count_clusters; 4616c9de560dSAlex Tomas int err = 0; 4617c9de560dSAlex Tomas int ret; 4618c9de560dSAlex Tomas 4619b10a44c3STheodore Ts'o might_sleep(); 4620e6362609STheodore Ts'o if (bh) { 4621e6362609STheodore Ts'o if (block) 4622e6362609STheodore Ts'o BUG_ON(block != bh->b_blocknr); 4623e6362609STheodore Ts'o else 4624e6362609STheodore Ts'o block = bh->b_blocknr; 4625e6362609STheodore Ts'o } 4626c9de560dSAlex Tomas 4627c9de560dSAlex Tomas sbi = EXT4_SB(sb); 46281f2acb60STheodore Ts'o if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 46291f2acb60STheodore Ts'o !ext4_data_block_valid(sbi, block, count)) { 463012062dddSEric Sandeen ext4_error(sb, "Freeing blocks not in datazone - " 46310610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 4632c9de560dSAlex Tomas goto error_return; 4633c9de560dSAlex Tomas } 4634c9de560dSAlex Tomas 46350610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 4636e6362609STheodore Ts'o trace_ext4_free_blocks(inode, block, count, flags); 4637e6362609STheodore Ts'o 4638e6362609STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_FORGET) { 4639e6362609STheodore Ts'o struct buffer_head *tbh = bh; 4640e6362609STheodore Ts'o int i; 4641e6362609STheodore Ts'o 4642e6362609STheodore Ts'o BUG_ON(bh && (count > 1)); 4643e6362609STheodore Ts'o 4644e6362609STheodore Ts'o for (i = 0; i < count; i++) { 46452ed5724dSTheodore Ts'o cond_resched(); 4646e6362609STheodore Ts'o if (!bh) 4647e6362609STheodore Ts'o tbh = sb_find_get_block(inode->i_sb, 4648e6362609STheodore Ts'o block + i); 46492ed5724dSTheodore Ts'o if (!tbh) 465087783690SNamhyung Kim continue; 4651e6362609STheodore Ts'o ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 4652e6362609STheodore Ts'o inode, tbh, block + i); 4653e6362609STheodore Ts'o } 4654e6362609STheodore Ts'o } 4655e6362609STheodore Ts'o 4656e6362609STheodore Ts'o /* 4657e6362609STheodore Ts'o * We need to make sure we don't reuse the freed block until 4658e6362609STheodore Ts'o * after the transaction is committed, which we can do by 4659e6362609STheodore Ts'o * treating the block as metadata, below. We make an 4660e6362609STheodore Ts'o * exception if the inode is to be written in writeback mode 4661e6362609STheodore Ts'o * since writeback mode has weak data consistency guarantees. 4662e6362609STheodore Ts'o */ 4663e6362609STheodore Ts'o if (!ext4_should_writeback_data(inode)) 4664e6362609STheodore Ts'o flags |= EXT4_FREE_BLOCKS_METADATA; 4665c9de560dSAlex Tomas 466684130193STheodore Ts'o /* 466784130193STheodore Ts'o * If the extent to be freed does not begin on a cluster 466884130193STheodore Ts'o * boundary, we need to deal with partial clusters at the 466984130193STheodore Ts'o * beginning and end of the extent. Normally we will free 467084130193STheodore Ts'o * blocks at the beginning or the end unless we are explicitly 467184130193STheodore Ts'o * requested to avoid doing so. 467284130193STheodore Ts'o */ 4673f5a44db5STheodore Ts'o overflow = EXT4_PBLK_COFF(sbi, block); 467484130193STheodore Ts'o if (overflow) { 467584130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_FIRST_CLUSTER) { 467684130193STheodore Ts'o overflow = sbi->s_cluster_ratio - overflow; 467784130193STheodore Ts'o block += overflow; 467884130193STheodore Ts'o if (count > overflow) 467984130193STheodore Ts'o count -= overflow; 468084130193STheodore Ts'o else 468184130193STheodore Ts'o return; 468284130193STheodore Ts'o } else { 468384130193STheodore Ts'o block -= overflow; 468484130193STheodore Ts'o count += overflow; 468584130193STheodore Ts'o } 468684130193STheodore Ts'o } 4687f5a44db5STheodore Ts'o overflow = EXT4_LBLK_COFF(sbi, count); 468884130193STheodore Ts'o if (overflow) { 468984130193STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_NOFREE_LAST_CLUSTER) { 469084130193STheodore Ts'o if (count > overflow) 469184130193STheodore Ts'o count -= overflow; 469284130193STheodore Ts'o else 469384130193STheodore Ts'o return; 469484130193STheodore Ts'o } else 469584130193STheodore Ts'o count += sbi->s_cluster_ratio - overflow; 469684130193STheodore Ts'o } 469784130193STheodore Ts'o 4698c9de560dSAlex Tomas do_more: 4699c9de560dSAlex Tomas overflow = 0; 4700c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 4701c9de560dSAlex Tomas 4702163a203dSDarrick J. Wong if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT( 4703163a203dSDarrick J. Wong ext4_get_group_info(sb, block_group)))) 4704163a203dSDarrick J. Wong return; 4705163a203dSDarrick J. Wong 4706c9de560dSAlex Tomas /* 4707c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 4708c9de560dSAlex Tomas * boundary. 4709c9de560dSAlex Tomas */ 471084130193STheodore Ts'o if (EXT4_C2B(sbi, bit) + count > EXT4_BLOCKS_PER_GROUP(sb)) { 471184130193STheodore Ts'o overflow = EXT4_C2B(sbi, bit) + count - 471284130193STheodore Ts'o EXT4_BLOCKS_PER_GROUP(sb); 4713c9de560dSAlex Tomas count -= overflow; 4714c9de560dSAlex Tomas } 4715810da240SLukas Czerner count_clusters = EXT4_NUM_B2C(sbi, count); 4716574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 4717ce89f46cSAneesh Kumar K.V if (!bitmap_bh) { 4718ce89f46cSAneesh Kumar K.V err = -EIO; 4719c9de560dSAlex Tomas goto error_return; 4720ce89f46cSAneesh Kumar K.V } 4721c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 4722ce89f46cSAneesh Kumar K.V if (!gdp) { 4723ce89f46cSAneesh Kumar K.V err = -EIO; 4724c9de560dSAlex Tomas goto error_return; 4725ce89f46cSAneesh Kumar K.V } 4726c9de560dSAlex Tomas 4727c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 4728c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 4729c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 4730c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group) || 4731c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 4732c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group)) { 4733c9de560dSAlex Tomas 473412062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 47350610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 4736519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 4737519deca0SAneesh Kumar K.V goto error_return; 4738c9de560dSAlex Tomas } 4739c9de560dSAlex Tomas 4740c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 4741c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 4742c9de560dSAlex Tomas if (err) 4743c9de560dSAlex Tomas goto error_return; 4744c9de560dSAlex Tomas 4745c9de560dSAlex Tomas /* 4746c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 4747c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 4748c9de560dSAlex Tomas * using it 4749c9de560dSAlex Tomas */ 4750c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 4751c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 4752c9de560dSAlex Tomas if (err) 4753c9de560dSAlex Tomas goto error_return; 4754c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4755c9de560dSAlex Tomas { 4756c9de560dSAlex Tomas int i; 475784130193STheodore Ts'o for (i = 0; i < count_clusters; i++) 4758c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 4759c9de560dSAlex Tomas } 4760c9de560dSAlex Tomas #endif 476184130193STheodore Ts'o trace_ext4_mballoc_free(sb, inode, block_group, bit, count_clusters); 4762c9de560dSAlex Tomas 4763920313a7SAneesh Kumar K.V err = ext4_mb_load_buddy(sb, block_group, &e4b); 4764920313a7SAneesh Kumar K.V if (err) 4765920313a7SAneesh Kumar K.V goto error_return; 4766e6362609STheodore Ts'o 4767e6362609STheodore Ts'o if ((flags & EXT4_FREE_BLOCKS_METADATA) && ext4_handle_valid(handle)) { 47687a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 47697a2fcbf7SAneesh Kumar K.V /* 47707a2fcbf7SAneesh Kumar K.V * blocks being freed are metadata. these blocks shouldn't 47717a2fcbf7SAneesh Kumar K.V * be used until this transaction is committed 47727a2fcbf7SAneesh Kumar K.V */ 4773e7676a70STheodore Ts'o retry: 477418aadd47SBobi Jam new_entry = kmem_cache_alloc(ext4_free_data_cachep, GFP_NOFS); 4775b72143abSTheodore Ts'o if (!new_entry) { 4776e7676a70STheodore Ts'o /* 4777e7676a70STheodore Ts'o * We use a retry loop because 4778e7676a70STheodore Ts'o * ext4_free_blocks() is not allowed to fail. 4779e7676a70STheodore Ts'o */ 4780e7676a70STheodore Ts'o cond_resched(); 4781e7676a70STheodore Ts'o congestion_wait(BLK_RW_ASYNC, HZ/50); 4782e7676a70STheodore Ts'o goto retry; 4783b72143abSTheodore Ts'o } 478418aadd47SBobi Jam new_entry->efd_start_cluster = bit; 478518aadd47SBobi Jam new_entry->efd_group = block_group; 478618aadd47SBobi Jam new_entry->efd_count = count_clusters; 478718aadd47SBobi Jam new_entry->efd_tid = handle->h_transaction->t_tid; 4788955ce5f5SAneesh Kumar K.V 47897a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 479084130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 47917a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 4792c9de560dSAlex Tomas } else { 47937a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 47947a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 47957a2fcbf7SAneesh Kumar K.V * them with group lock_held 47967a2fcbf7SAneesh Kumar K.V */ 4797d71c1ae2SLukas Czerner if (test_opt(sb, DISCARD)) { 4798d71c1ae2SLukas Czerner err = ext4_issue_discard(sb, block_group, bit, count); 4799d71c1ae2SLukas Czerner if (err && err != -EOPNOTSUPP) 4800d71c1ae2SLukas Czerner ext4_msg(sb, KERN_WARNING, "discard request in" 4801d71c1ae2SLukas Czerner " group:%d block:%d count:%lu failed" 4802d71c1ae2SLukas Czerner " with %d", block_group, bit, count, 4803d71c1ae2SLukas Czerner err); 48048f9ff189SLukas Czerner } else 48058f9ff189SLukas Czerner EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info); 4806d71c1ae2SLukas Czerner 4807955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 480884130193STheodore Ts'o mb_clear_bits(bitmap_bh->b_data, bit, count_clusters); 480984130193STheodore Ts'o mb_free_blocks(inode, &e4b, bit, count_clusters); 4810c9de560dSAlex Tomas } 4811c9de560dSAlex Tomas 4812021b65bbSTheodore Ts'o ret = ext4_free_group_clusters(sb, gdp) + count_clusters; 4813021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, gdp, ret); 481479f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, gdp, bitmap_bh); 4815feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, gdp); 4816955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 4817c9de560dSAlex Tomas 4818772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 4819772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 482090ba983fSTheodore Ts'o atomic64_add(count_clusters, 482124aaa8efSTheodore Ts'o &sbi->s_flex_groups[flex_group].free_clusters); 4822772cb7c8SJose R. Santos } 4823772cb7c8SJose R. Santos 48247d734532SJan Kara if (flags & EXT4_FREE_BLOCKS_RESERVE && ei->i_reserved_data_blocks) { 48257d734532SJan Kara percpu_counter_add(&sbi->s_dirtyclusters_counter, 48267d734532SJan Kara count_clusters); 48277d734532SJan Kara spin_lock(&ei->i_block_reservation_lock); 48287d734532SJan Kara if (flags & EXT4_FREE_BLOCKS_METADATA) 48297d734532SJan Kara ei->i_reserved_meta_blocks += count_clusters; 48307d734532SJan Kara else 48317d734532SJan Kara ei->i_reserved_data_blocks += count_clusters; 48327d734532SJan Kara spin_unlock(&ei->i_block_reservation_lock); 48337b415bf6SAditya Kali if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 48347d734532SJan Kara dquot_reclaim_block(inode, 48357d734532SJan Kara EXT4_C2B(sbi, count_clusters)); 48367d734532SJan Kara } else if (!(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 48377b415bf6SAditya Kali dquot_free_block(inode, EXT4_C2B(sbi, count_clusters)); 48387d734532SJan Kara percpu_counter_add(&sbi->s_freeclusters_counter, count_clusters); 48397d734532SJan Kara 48407d734532SJan Kara ext4_mb_unload_buddy(&e4b); 48417b415bf6SAditya Kali 48427a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 48437a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 48447a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 48457a2fcbf7SAneesh Kumar K.V 4846c9de560dSAlex Tomas /* And the group descriptor block */ 4847c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 48480390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 4849c9de560dSAlex Tomas if (!err) 4850c9de560dSAlex Tomas err = ret; 4851c9de560dSAlex Tomas 4852c9de560dSAlex Tomas if (overflow && !err) { 4853c9de560dSAlex Tomas block += count; 4854c9de560dSAlex Tomas count = overflow; 4855c9de560dSAlex Tomas put_bh(bitmap_bh); 4856c9de560dSAlex Tomas goto do_more; 4857c9de560dSAlex Tomas } 4858c9de560dSAlex Tomas error_return: 4859c9de560dSAlex Tomas brelse(bitmap_bh); 4860c9de560dSAlex Tomas ext4_std_error(sb, err); 4861c9de560dSAlex Tomas return; 4862c9de560dSAlex Tomas } 48637360d173SLukas Czerner 48647360d173SLukas Czerner /** 48650529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 48662846e820SAmir Goldstein * @handle: handle to this transaction 48672846e820SAmir Goldstein * @sb: super block 48684907cb7bSAnatol Pomozov * @block: start physical block to add to the block group 48692846e820SAmir Goldstein * @count: number of blocks to free 48702846e820SAmir Goldstein * 4871e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 48722846e820SAmir Goldstein */ 4873cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 48742846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 48752846e820SAmir Goldstein { 48762846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 48772846e820SAmir Goldstein struct buffer_head *gd_bh; 48782846e820SAmir Goldstein ext4_group_t block_group; 48792846e820SAmir Goldstein ext4_grpblk_t bit; 48802846e820SAmir Goldstein unsigned int i; 48812846e820SAmir Goldstein struct ext4_group_desc *desc; 48822846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 4883e73a347bSAmir Goldstein struct ext4_buddy e4b; 48842846e820SAmir Goldstein int err = 0, ret, blk_free_count; 48852846e820SAmir Goldstein ext4_grpblk_t blocks_freed; 48862846e820SAmir Goldstein 48872846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 48882846e820SAmir Goldstein 48894740b830SYongqiang Yang if (count == 0) 48904740b830SYongqiang Yang return 0; 48914740b830SYongqiang Yang 48922846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 48932846e820SAmir Goldstein /* 48942846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 48952846e820SAmir Goldstein * boundary. 48962846e820SAmir Goldstein */ 4897cc7365dfSYongqiang Yang if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) { 4898cc7365dfSYongqiang Yang ext4_warning(sb, "too much blocks added to group %u\n", 4899cc7365dfSYongqiang Yang block_group); 4900cc7365dfSYongqiang Yang err = -EINVAL; 49012846e820SAmir Goldstein goto error_return; 4902cc7365dfSYongqiang Yang } 49032cd05cc3STheodore Ts'o 49042846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 4905cc7365dfSYongqiang Yang if (!bitmap_bh) { 4906cc7365dfSYongqiang Yang err = -EIO; 49072846e820SAmir Goldstein goto error_return; 4908cc7365dfSYongqiang Yang } 4909cc7365dfSYongqiang Yang 49102846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 4911cc7365dfSYongqiang Yang if (!desc) { 4912cc7365dfSYongqiang Yang err = -EIO; 49132846e820SAmir Goldstein goto error_return; 4914cc7365dfSYongqiang Yang } 49152846e820SAmir Goldstein 49162846e820SAmir Goldstein if (in_range(ext4_block_bitmap(sb, desc), block, count) || 49172846e820SAmir Goldstein in_range(ext4_inode_bitmap(sb, desc), block, count) || 49182846e820SAmir Goldstein in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) || 49192846e820SAmir Goldstein in_range(block + count - 1, ext4_inode_table(sb, desc), 49202846e820SAmir Goldstein sbi->s_itb_per_group)) { 49212846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 49222846e820SAmir Goldstein "Block = %llu, count = %lu", 49232846e820SAmir Goldstein block, count); 4924cc7365dfSYongqiang Yang err = -EINVAL; 49252846e820SAmir Goldstein goto error_return; 49262846e820SAmir Goldstein } 49272846e820SAmir Goldstein 49282cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 49292cd05cc3STheodore Ts'o err = ext4_journal_get_write_access(handle, bitmap_bh); 49302846e820SAmir Goldstein if (err) 49312846e820SAmir Goldstein goto error_return; 49322846e820SAmir Goldstein 49332846e820SAmir Goldstein /* 49342846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 49352846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 49362846e820SAmir Goldstein * using it 49372846e820SAmir Goldstein */ 49382846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 49392846e820SAmir Goldstein err = ext4_journal_get_write_access(handle, gd_bh); 49402846e820SAmir Goldstein if (err) 49412846e820SAmir Goldstein goto error_return; 4942e73a347bSAmir Goldstein 49432846e820SAmir Goldstein for (i = 0, blocks_freed = 0; i < count; i++) { 49442846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 4945e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 49462846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 49472846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 49482846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 49492846e820SAmir Goldstein } else { 49502846e820SAmir Goldstein blocks_freed++; 49512846e820SAmir Goldstein } 49522846e820SAmir Goldstein } 4953e73a347bSAmir Goldstein 4954e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 4955e73a347bSAmir Goldstein if (err) 4956e73a347bSAmir Goldstein goto error_return; 4957e73a347bSAmir Goldstein 4958e73a347bSAmir Goldstein /* 4959e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 4960e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 4961e73a347bSAmir Goldstein * them with group lock_held 4962e73a347bSAmir Goldstein */ 49632846e820SAmir Goldstein ext4_lock_group(sb, block_group); 4964e73a347bSAmir Goldstein mb_clear_bits(bitmap_bh->b_data, bit, count); 4965e73a347bSAmir Goldstein mb_free_blocks(NULL, &e4b, bit, count); 4966021b65bbSTheodore Ts'o blk_free_count = blocks_freed + ext4_free_group_clusters(sb, desc); 4967021b65bbSTheodore Ts'o ext4_free_group_clusters_set(sb, desc, blk_free_count); 496879f1ba49STao Ma ext4_block_bitmap_csum_set(sb, block_group, desc, bitmap_bh); 4969feb0ab32SDarrick J. Wong ext4_group_desc_csum_set(sb, block_group, desc); 49702846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 497157042651STheodore Ts'o percpu_counter_add(&sbi->s_freeclusters_counter, 4972810da240SLukas Czerner EXT4_NUM_B2C(sbi, blocks_freed)); 49732846e820SAmir Goldstein 49742846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 49752846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 497690ba983fSTheodore Ts'o atomic64_add(EXT4_NUM_B2C(sbi, blocks_freed), 497724aaa8efSTheodore Ts'o &sbi->s_flex_groups[flex_group].free_clusters); 49782846e820SAmir Goldstein } 4979e73a347bSAmir Goldstein 4980e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 49812846e820SAmir Goldstein 49822846e820SAmir Goldstein /* We dirtied the bitmap block */ 49832846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 49842846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 49852846e820SAmir Goldstein 49862846e820SAmir Goldstein /* And the group descriptor block */ 49872846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 49882846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 49892846e820SAmir Goldstein if (!err) 49902846e820SAmir Goldstein err = ret; 49912846e820SAmir Goldstein 49922846e820SAmir Goldstein error_return: 49932846e820SAmir Goldstein brelse(bitmap_bh); 49942846e820SAmir Goldstein ext4_std_error(sb, err); 4995cc7365dfSYongqiang Yang return err; 49962846e820SAmir Goldstein } 49972846e820SAmir Goldstein 49982846e820SAmir Goldstein /** 49997360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 50007360d173SLukas Czerner * @sb: super block for the file system 50017360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 50027360d173SLukas Czerner * @count: number of blocks to TRIM 50037360d173SLukas Czerner * @group: alloc. group we are working with 50047360d173SLukas Czerner * @e4b: ext4 buddy for the group 50057360d173SLukas Czerner * 50067360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 50077360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 50087360d173SLukas Czerner * be called with under the group lock. 50097360d173SLukas Czerner */ 5010d71c1ae2SLukas Czerner static int ext4_trim_extent(struct super_block *sb, int start, int count, 50117360d173SLukas Czerner ext4_group_t group, struct ext4_buddy *e4b) 50127360d173SLukas Czerner { 50137360d173SLukas Czerner struct ext4_free_extent ex; 5014d71c1ae2SLukas Czerner int ret = 0; 50157360d173SLukas Czerner 5016b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 5017b3d4c2b1STao Ma 50187360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 50197360d173SLukas Czerner 50207360d173SLukas Czerner ex.fe_start = start; 50217360d173SLukas Czerner ex.fe_group = group; 50227360d173SLukas Czerner ex.fe_len = count; 50237360d173SLukas Czerner 50247360d173SLukas Czerner /* 50257360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 50267360d173SLukas Czerner * being trimmed. 50277360d173SLukas Czerner */ 50287360d173SLukas Czerner mb_mark_used(e4b, &ex); 50297360d173SLukas Czerner ext4_unlock_group(sb, group); 5030d71c1ae2SLukas Czerner ret = ext4_issue_discard(sb, group, start, count); 50317360d173SLukas Czerner ext4_lock_group(sb, group); 50327360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 5033d71c1ae2SLukas Czerner return ret; 50347360d173SLukas Czerner } 50357360d173SLukas Czerner 50367360d173SLukas Czerner /** 50377360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 50387360d173SLukas Czerner * @sb: super block for file system 503922612283STao Ma * @group: group to be trimmed 50407360d173SLukas Czerner * @start: first group block to examine 50417360d173SLukas Czerner * @max: last group block to examine 50427360d173SLukas Czerner * @minblocks: minimum extent block count 50437360d173SLukas Czerner * 50447360d173SLukas Czerner * ext4_trim_all_free walks through group's buddy bitmap searching for free 50457360d173SLukas Czerner * extents. When the free block is found, ext4_trim_extent is called to TRIM 50467360d173SLukas Czerner * the extent. 50477360d173SLukas Czerner * 50487360d173SLukas Czerner * 50497360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 50507360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 50517360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 50527360d173SLukas Czerner * the group buddy bitmap. This is done until whole group is scanned. 50537360d173SLukas Czerner */ 50540b75a840SLukas Czerner static ext4_grpblk_t 505578944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 505678944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 505778944086SLukas Czerner ext4_grpblk_t minblocks) 50587360d173SLukas Czerner { 50597360d173SLukas Czerner void *bitmap; 5060169ddc3eSTao Ma ext4_grpblk_t next, count = 0, free_count = 0; 506178944086SLukas Czerner struct ext4_buddy e4b; 5062d71c1ae2SLukas Czerner int ret = 0; 50637360d173SLukas Czerner 5064b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 5065b3d4c2b1STao Ma 506678944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 506778944086SLukas Czerner if (ret) { 506878944086SLukas Czerner ext4_error(sb, "Error in loading buddy " 506978944086SLukas Czerner "information for %u", group); 507078944086SLukas Czerner return ret; 507178944086SLukas Czerner } 507278944086SLukas Czerner bitmap = e4b.bd_bitmap; 507328739eeaSLukas Czerner 507428739eeaSLukas Czerner ext4_lock_group(sb, group); 50753d56b8d2STao Ma if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) && 50763d56b8d2STao Ma minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) 50773d56b8d2STao Ma goto out; 50783d56b8d2STao Ma 507978944086SLukas Czerner start = (e4b.bd_info->bb_first_free > start) ? 508078944086SLukas Czerner e4b.bd_info->bb_first_free : start; 50817360d173SLukas Czerner 5082913eed83SLukas Czerner while (start <= max) { 5083913eed83SLukas Czerner start = mb_find_next_zero_bit(bitmap, max + 1, start); 5084913eed83SLukas Czerner if (start > max) 50857360d173SLukas Czerner break; 5086913eed83SLukas Czerner next = mb_find_next_bit(bitmap, max + 1, start); 50877360d173SLukas Czerner 50887360d173SLukas Czerner if ((next - start) >= minblocks) { 5089d71c1ae2SLukas Czerner ret = ext4_trim_extent(sb, start, 509078944086SLukas Czerner next - start, group, &e4b); 5091d71c1ae2SLukas Czerner if (ret && ret != -EOPNOTSUPP) 5092d71c1ae2SLukas Czerner break; 5093d71c1ae2SLukas Czerner ret = 0; 50947360d173SLukas Czerner count += next - start; 50957360d173SLukas Czerner } 5096169ddc3eSTao Ma free_count += next - start; 50977360d173SLukas Czerner start = next + 1; 50987360d173SLukas Czerner 50997360d173SLukas Czerner if (fatal_signal_pending(current)) { 51007360d173SLukas Czerner count = -ERESTARTSYS; 51017360d173SLukas Czerner break; 51027360d173SLukas Czerner } 51037360d173SLukas Czerner 51047360d173SLukas Czerner if (need_resched()) { 51057360d173SLukas Czerner ext4_unlock_group(sb, group); 51067360d173SLukas Czerner cond_resched(); 51077360d173SLukas Czerner ext4_lock_group(sb, group); 51087360d173SLukas Czerner } 51097360d173SLukas Czerner 5110169ddc3eSTao Ma if ((e4b.bd_info->bb_free - free_count) < minblocks) 51117360d173SLukas Czerner break; 51127360d173SLukas Czerner } 51133d56b8d2STao Ma 5114d71c1ae2SLukas Czerner if (!ret) { 5115d71c1ae2SLukas Czerner ret = count; 51163d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 5117d71c1ae2SLukas Czerner } 51183d56b8d2STao Ma out: 51197360d173SLukas Czerner ext4_unlock_group(sb, group); 512078944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 51217360d173SLukas Czerner 51227360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 51237360d173SLukas Czerner count, group); 51247360d173SLukas Czerner 5125d71c1ae2SLukas Czerner return ret; 51267360d173SLukas Czerner } 51277360d173SLukas Czerner 51287360d173SLukas Czerner /** 51297360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 51307360d173SLukas Czerner * @sb: superblock for filesystem 51317360d173SLukas Czerner * @range: fstrim_range structure 51327360d173SLukas Czerner * 51337360d173SLukas Czerner * start: First Byte to trim 51347360d173SLukas Czerner * len: number of Bytes to trim from start 51357360d173SLukas Czerner * minlen: minimum extent length in Bytes 51367360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 51377360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 51387360d173SLukas Czerner * is invoked to trim all free space. 51397360d173SLukas Czerner */ 51407360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 51417360d173SLukas Czerner { 514278944086SLukas Czerner struct ext4_group_info *grp; 5143913eed83SLukas Czerner ext4_group_t group, first_group, last_group; 51447137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 5145913eed83SLukas Czerner uint64_t start, end, minlen, trimmed = 0; 51460f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 51470f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 5148913eed83SLukas Czerner ext4_fsblk_t max_blks = ext4_blocks_count(EXT4_SB(sb)->s_es); 51497360d173SLukas Czerner int ret = 0; 51507360d173SLukas Czerner 51517360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 5152913eed83SLukas Czerner end = start + (range->len >> sb->s_blocksize_bits) - 1; 5153aaf7d73eSLukas Czerner minlen = EXT4_NUM_B2C(EXT4_SB(sb), 5154aaf7d73eSLukas Czerner range->minlen >> sb->s_blocksize_bits); 51557360d173SLukas Czerner 51565de35e8dSLukas Czerner if (minlen > EXT4_CLUSTERS_PER_GROUP(sb) || 51575de35e8dSLukas Czerner start >= max_blks || 51585de35e8dSLukas Czerner range->len < sb->s_blocksize) 51597360d173SLukas Czerner return -EINVAL; 5160913eed83SLukas Czerner if (end >= max_blks) 5161913eed83SLukas Czerner end = max_blks - 1; 5162913eed83SLukas Czerner if (end <= first_data_blk) 516322f10457STao Ma goto out; 5164913eed83SLukas Czerner if (start < first_data_blk) 51650f0a25bfSJan Kara start = first_data_blk; 51667360d173SLukas Czerner 5167913eed83SLukas Czerner /* Determine first and last group to examine based on start and end */ 51687360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 51697137d7a4STheodore Ts'o &first_group, &first_cluster); 5170913eed83SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) end, 51717137d7a4STheodore Ts'o &last_group, &last_cluster); 51727360d173SLukas Czerner 5173913eed83SLukas Czerner /* end now represents the last cluster to discard in this group */ 5174913eed83SLukas Czerner end = EXT4_CLUSTERS_PER_GROUP(sb) - 1; 51757360d173SLukas Czerner 51767360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 517778944086SLukas Czerner grp = ext4_get_group_info(sb, group); 517878944086SLukas Czerner /* We only do this if the grp has never been initialized */ 517978944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 518078944086SLukas Czerner ret = ext4_mb_init_group(sb, group); 518178944086SLukas Czerner if (ret) 51827360d173SLukas Czerner break; 51837360d173SLukas Czerner } 51847360d173SLukas Czerner 51850ba08517STao Ma /* 5186913eed83SLukas Czerner * For all the groups except the last one, last cluster will 5187913eed83SLukas Czerner * always be EXT4_CLUSTERS_PER_GROUP(sb)-1, so we only need to 5188913eed83SLukas Czerner * change it for the last group, note that last_cluster is 5189913eed83SLukas Czerner * already computed earlier by ext4_get_group_no_and_offset() 51900ba08517STao Ma */ 5191913eed83SLukas Czerner if (group == last_group) 5192913eed83SLukas Czerner end = last_cluster; 51937360d173SLukas Czerner 519478944086SLukas Czerner if (grp->bb_free >= minlen) { 51957137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 5196913eed83SLukas Czerner end, minlen); 51977360d173SLukas Czerner if (cnt < 0) { 51987360d173SLukas Czerner ret = cnt; 51997360d173SLukas Czerner break; 52007360d173SLukas Czerner } 52017360d173SLukas Czerner trimmed += cnt; 520221e7fd22SLukas Czerner } 5203913eed83SLukas Czerner 5204913eed83SLukas Czerner /* 5205913eed83SLukas Czerner * For every group except the first one, we are sure 5206913eed83SLukas Czerner * that the first cluster to discard will be cluster #0. 5207913eed83SLukas Czerner */ 52087137d7a4STheodore Ts'o first_cluster = 0; 52097360d173SLukas Czerner } 52107360d173SLukas Czerner 52113d56b8d2STao Ma if (!ret) 52123d56b8d2STao Ma atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); 52133d56b8d2STao Ma 521422f10457STao Ma out: 5215aaf7d73eSLukas Czerner range->len = EXT4_C2B(EXT4_SB(sb), trimmed) << sb->s_blocksize_bits; 52167360d173SLukas Czerner return ret; 52177360d173SLukas Czerner } 5218