1c9de560dSAlex Tomas /* 2c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 3c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 4c9de560dSAlex Tomas * 5c9de560dSAlex Tomas * This program is free software; you can redistribute it and/or modify 6c9de560dSAlex Tomas * it under the terms of the GNU General Public License version 2 as 7c9de560dSAlex Tomas * published by the Free Software Foundation. 8c9de560dSAlex Tomas * 9c9de560dSAlex Tomas * This program is distributed in the hope that it will be useful, 10c9de560dSAlex Tomas * but WITHOUT ANY WARRANTY; without even the implied warranty of 11c9de560dSAlex Tomas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12c9de560dSAlex Tomas * GNU General Public License for more details. 13c9de560dSAlex Tomas * 14c9de560dSAlex Tomas * You should have received a copy of the GNU General Public Licens 15c9de560dSAlex Tomas * along with this program; if not, write to the Free Software 16c9de560dSAlex Tomas * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- 17c9de560dSAlex Tomas */ 18c9de560dSAlex Tomas 19c9de560dSAlex Tomas 20c9de560dSAlex Tomas /* 21c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 22c9de560dSAlex Tomas */ 23c9de560dSAlex Tomas 248f6e39a7SMingming Cao #include "mballoc.h" 256ba495e9STheodore Ts'o #include <linux/debugfs.h> 269bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 279bffad1eSTheodore Ts'o 28c9de560dSAlex Tomas /* 29c9de560dSAlex Tomas * MUSTDO: 30c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 31c9de560dSAlex Tomas * - search for metadata in few groups 32c9de560dSAlex Tomas * 33c9de560dSAlex Tomas * TODO v4: 34c9de560dSAlex Tomas * - normalization should take into account whether file is still open 35c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 36c9de560dSAlex Tomas * - don't normalize tails 37c9de560dSAlex Tomas * - quota 38c9de560dSAlex Tomas * - reservation for superuser 39c9de560dSAlex Tomas * 40c9de560dSAlex Tomas * TODO v3: 41c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 42c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 43c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 44c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 45c9de560dSAlex Tomas * - error handling 46c9de560dSAlex Tomas */ 47c9de560dSAlex Tomas 48c9de560dSAlex Tomas /* 49c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 50c9de560dSAlex Tomas * near to the goal(block) value specified. 51c9de560dSAlex Tomas * 52b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 53b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 54b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 55b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 56b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 57b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 58b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 59b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 60b713a5ecSTheodore Ts'o * terms of number of blocks. 61c9de560dSAlex Tomas * 62c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 63b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 64c9de560dSAlex Tomas * 65b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 66b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 67b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 68b713a5ecSTheodore Ts'o * represented as: 69c9de560dSAlex Tomas * 70c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 71c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 72c9de560dSAlex Tomas * pa_len -> lenght for this prealloc space 73c9de560dSAlex Tomas * pa_free -> free space available in this prealloc space 74c9de560dSAlex Tomas * 75c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 76c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 77c9de560dSAlex Tomas * space we will consume the particular prealloc space. This make sure that 78c9de560dSAlex Tomas * that the we have contiguous physical blocks representing the file blocks 79c9de560dSAlex Tomas * 80c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 81c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 82c9de560dSAlex Tomas * pa_free. 83c9de560dSAlex Tomas * 84c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 85c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 86c9de560dSAlex Tomas * prealloc space. These are per CPU prealloc list repreasented as 87c9de560dSAlex Tomas * 88c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 89c9de560dSAlex Tomas * 90c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 91c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 92c9de560dSAlex Tomas * 93c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 94c9de560dSAlex Tomas * enough free space (pa_free) withing the prealloc space. 95c9de560dSAlex Tomas * 96c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 97c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 98c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 99c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 100c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 101c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 102c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 103c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 104c9de560dSAlex Tomas * inode as: 105c9de560dSAlex Tomas * 106c9de560dSAlex Tomas * { page } 107c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 108c9de560dSAlex Tomas * 109c9de560dSAlex Tomas * 110c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 111c9de560dSAlex Tomas * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE / 112c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 113c9de560dSAlex Tomas * which is blocks_per_page/2 114c9de560dSAlex Tomas * 115c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 116c9de560dSAlex Tomas * away when the filesystem is unmounted. 117c9de560dSAlex Tomas * 118c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 119c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 120c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 121c9de560dSAlex Tomas * 122c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 123c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 124c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 125c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 126c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 127c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 128b713a5ecSTheodore Ts'o * sbi->s_mb_group_prealloc. Default value of s_mb_group_prealloc is 129c9de560dSAlex Tomas * 512 blocks. This can be tuned via 130b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition/mb_group_prealloc. The value is represented in 131c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 132c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 133c9de560dSAlex Tomas * stripe value (sbi->s_stripe) 134c9de560dSAlex Tomas * 135b713a5ecSTheodore Ts'o * The regular allocator(using the buddy cache) supports few tunables. 136c9de560dSAlex Tomas * 137b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 138b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 139b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 140c9de560dSAlex Tomas * 141b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 142c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 143c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 144b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 145c9de560dSAlex Tomas * stripe size (sbi->s_stripe), we try to search for contigous block in 146b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 147b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 148b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 149c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 150b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 151c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 152c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 153c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 154c9de560dSAlex Tomas * can used for allocation. ext4_mb_good_group explains how the groups are 155c9de560dSAlex Tomas * checked. 156c9de560dSAlex Tomas * 157c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 158c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 159c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 160c9de560dSAlex Tomas * subsequent request. 161c9de560dSAlex Tomas */ 162c9de560dSAlex Tomas 163c9de560dSAlex Tomas /* 164c9de560dSAlex Tomas * mballoc operates on the following data: 165c9de560dSAlex Tomas * - on-disk bitmap 166c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 167c9de560dSAlex Tomas * - preallocation descriptors (PAs) 168c9de560dSAlex Tomas * 169c9de560dSAlex Tomas * there are two types of preallocations: 170c9de560dSAlex Tomas * - inode 171c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 172c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 173c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 174c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 175c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 176c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 177c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 178c9de560dSAlex Tomas * must discard all preallocated blocks. 179c9de560dSAlex Tomas * - locality group 180c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 181c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 182c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 183c9de560dSAlex Tomas * it's consumed from the beginning to the end. 184c9de560dSAlex Tomas * 185c9de560dSAlex Tomas * relation between them can be expressed as: 186c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 187c9de560dSAlex Tomas * 188c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 189c9de560dSAlex Tomas * - allocated blocks (persistent) 190c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 191c9de560dSAlex Tomas * 192c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 193c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 194c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 195c9de560dSAlex Tomas * 196c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 197c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 198c9de560dSAlex Tomas * 199c9de560dSAlex Tomas * all operations can be expressed as: 200c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 201c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 202c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 203c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 204c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 205c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 206c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 207c9de560dSAlex Tomas * is used in real operation because we can't know actual used 208c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 209c9de560dSAlex Tomas * 210c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 211c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 212c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 213c9de560dSAlex Tomas * the following knowledge: 214c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 215c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 216c9de560dSAlex Tomas * nobody can re-allocate that block 217c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 218c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 219c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 220c9de560dSAlex Tomas * block 221c9de560dSAlex Tomas * 222c9de560dSAlex Tomas * so, now we're building a concurrency table: 223c9de560dSAlex Tomas * - init buddy vs. 224c9de560dSAlex Tomas * - new PA 225c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 226c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 227c9de560dSAlex Tomas * - use inode PA 228c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 229c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 230c9de560dSAlex Tomas * - discard inode PA 231c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 232c9de560dSAlex Tomas * - use locality group PA 233c9de560dSAlex Tomas * again PA-=N must be serialized with init 234c9de560dSAlex Tomas * - discard locality group PA 235c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 236c9de560dSAlex Tomas * - new PA vs. 237c9de560dSAlex Tomas * - use inode PA 238c9de560dSAlex Tomas * i_data_sem serializes them 239c9de560dSAlex Tomas * - discard inode PA 240c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 241c9de560dSAlex Tomas * - use locality group PA 242c9de560dSAlex Tomas * some mutex should serialize them 243c9de560dSAlex Tomas * - discard locality group PA 244c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 245c9de560dSAlex Tomas * - use inode PA 246c9de560dSAlex Tomas * - use inode PA 247c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 248c9de560dSAlex Tomas * - discard inode PA 249c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 250c9de560dSAlex Tomas * - use locality group PA 251c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 252c9de560dSAlex Tomas * - discard locality group PA 253c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 254c9de560dSAlex Tomas * 255c9de560dSAlex Tomas * now we're ready to make few consequences: 256c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 257c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 258c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 259c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 260c9de560dSAlex Tomas * any discard or they're serialized somehow 261c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 262c9de560dSAlex Tomas * 263c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 264c9de560dSAlex Tomas * in this case, but we should care about concurrent init 265c9de560dSAlex Tomas * 266c9de560dSAlex Tomas */ 267c9de560dSAlex Tomas 268c9de560dSAlex Tomas /* 269c9de560dSAlex Tomas * Logic in few words: 270c9de560dSAlex Tomas * 271c9de560dSAlex Tomas * - allocation: 272c9de560dSAlex Tomas * load group 273c9de560dSAlex Tomas * find blocks 274c9de560dSAlex Tomas * mark bits in on-disk bitmap 275c9de560dSAlex Tomas * release group 276c9de560dSAlex Tomas * 277c9de560dSAlex Tomas * - use preallocation: 278c9de560dSAlex Tomas * find proper PA (per-inode or group) 279c9de560dSAlex Tomas * load group 280c9de560dSAlex Tomas * mark bits in on-disk bitmap 281c9de560dSAlex Tomas * release group 282c9de560dSAlex Tomas * release PA 283c9de560dSAlex Tomas * 284c9de560dSAlex Tomas * - free: 285c9de560dSAlex Tomas * load group 286c9de560dSAlex Tomas * mark bits in on-disk bitmap 287c9de560dSAlex Tomas * release group 288c9de560dSAlex Tomas * 289c9de560dSAlex Tomas * - discard preallocations in group: 290c9de560dSAlex Tomas * mark PAs deleted 291c9de560dSAlex Tomas * move them onto local list 292c9de560dSAlex Tomas * load on-disk bitmap 293c9de560dSAlex Tomas * load group 294c9de560dSAlex Tomas * remove PA from object (inode or locality group) 295c9de560dSAlex Tomas * mark free blocks in-core 296c9de560dSAlex Tomas * 297c9de560dSAlex Tomas * - discard inode's preallocations: 298c9de560dSAlex Tomas */ 299c9de560dSAlex Tomas 300c9de560dSAlex Tomas /* 301c9de560dSAlex Tomas * Locking rules 302c9de560dSAlex Tomas * 303c9de560dSAlex Tomas * Locks: 304c9de560dSAlex Tomas * - bitlock on a group (group) 305c9de560dSAlex Tomas * - object (inode/locality) (object) 306c9de560dSAlex Tomas * - per-pa lock (pa) 307c9de560dSAlex Tomas * 308c9de560dSAlex Tomas * Paths: 309c9de560dSAlex Tomas * - new pa 310c9de560dSAlex Tomas * object 311c9de560dSAlex Tomas * group 312c9de560dSAlex Tomas * 313c9de560dSAlex Tomas * - find and use pa: 314c9de560dSAlex Tomas * pa 315c9de560dSAlex Tomas * 316c9de560dSAlex Tomas * - release consumed pa: 317c9de560dSAlex Tomas * pa 318c9de560dSAlex Tomas * group 319c9de560dSAlex Tomas * object 320c9de560dSAlex Tomas * 321c9de560dSAlex Tomas * - generate in-core bitmap: 322c9de560dSAlex Tomas * group 323c9de560dSAlex Tomas * pa 324c9de560dSAlex Tomas * 325c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 326c9de560dSAlex Tomas * object 327c9de560dSAlex Tomas * pa 328c9de560dSAlex Tomas * group 329c9de560dSAlex Tomas * 330c9de560dSAlex Tomas * - discard all for given group: 331c9de560dSAlex Tomas * group 332c9de560dSAlex Tomas * pa 333c9de560dSAlex Tomas * group 334c9de560dSAlex Tomas * object 335c9de560dSAlex Tomas * 336c9de560dSAlex Tomas */ 337c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 338c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 339c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_free_ext_cachep; 340c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 341c3a326a6SAneesh Kumar K.V ext4_group_t group); 3427a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 3437a2fcbf7SAneesh Kumar K.V ext4_group_t group); 344c3a326a6SAneesh Kumar K.V static void release_blocks_on_commit(journal_t *journal, transaction_t *txn); 345c3a326a6SAneesh Kumar K.V 346ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 347ffad0a44SAneesh Kumar K.V { 348c9de560dSAlex Tomas #if BITS_PER_LONG == 64 349ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 350ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 351c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 352ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 353ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 354c9de560dSAlex Tomas #else 355c9de560dSAlex Tomas #error "how many bits you are?!" 356c9de560dSAlex Tomas #endif 357ffad0a44SAneesh Kumar K.V return addr; 358ffad0a44SAneesh Kumar K.V } 359c9de560dSAlex Tomas 360c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 361c9de560dSAlex Tomas { 362c9de560dSAlex Tomas /* 363c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 364c9de560dSAlex Tomas * needs unsigned long aligned address 365c9de560dSAlex Tomas */ 366ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 367c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 368c9de560dSAlex Tomas } 369c9de560dSAlex Tomas 370c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 371c9de560dSAlex Tomas { 372ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 373c9de560dSAlex Tomas ext4_set_bit(bit, addr); 374c9de560dSAlex Tomas } 375c9de560dSAlex Tomas 376c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 377c9de560dSAlex Tomas { 378ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 379c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 380c9de560dSAlex Tomas } 381c9de560dSAlex Tomas 382ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 383ffad0a44SAneesh Kumar K.V { 384e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 385ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 386e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 387ffad0a44SAneesh Kumar K.V start += fix; 388ffad0a44SAneesh Kumar K.V 389e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 390e7dfb246SAneesh Kumar K.V if (ret > max) 391e7dfb246SAneesh Kumar K.V return max; 392e7dfb246SAneesh Kumar K.V return ret; 393ffad0a44SAneesh Kumar K.V } 394ffad0a44SAneesh Kumar K.V 395ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 396ffad0a44SAneesh Kumar K.V { 397e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 398ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 399e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 400ffad0a44SAneesh Kumar K.V start += fix; 401ffad0a44SAneesh Kumar K.V 402e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 403e7dfb246SAneesh Kumar K.V if (ret > max) 404e7dfb246SAneesh Kumar K.V return max; 405e7dfb246SAneesh Kumar K.V return ret; 406ffad0a44SAneesh Kumar K.V } 407ffad0a44SAneesh Kumar K.V 408c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 409c9de560dSAlex Tomas { 410c9de560dSAlex Tomas char *bb; 411c9de560dSAlex Tomas 412c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 413c9de560dSAlex Tomas BUG_ON(max == NULL); 414c9de560dSAlex Tomas 415c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 416c9de560dSAlex Tomas *max = 0; 417c9de560dSAlex Tomas return NULL; 418c9de560dSAlex Tomas } 419c9de560dSAlex Tomas 420c9de560dSAlex Tomas /* at order 0 we see each particular block */ 421c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 422c9de560dSAlex Tomas if (order == 0) 423c9de560dSAlex Tomas return EXT4_MB_BITMAP(e4b); 424c9de560dSAlex Tomas 425c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b) + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 426c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 427c9de560dSAlex Tomas 428c9de560dSAlex Tomas return bb; 429c9de560dSAlex Tomas } 430c9de560dSAlex Tomas 431c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 432c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 433c9de560dSAlex Tomas int first, int count) 434c9de560dSAlex Tomas { 435c9de560dSAlex Tomas int i; 436c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 437c9de560dSAlex Tomas 438c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 439c9de560dSAlex Tomas return; 440bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 441c9de560dSAlex Tomas for (i = 0; i < count; i++) { 442c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 443c9de560dSAlex Tomas ext4_fsblk_t blocknr; 444c9de560dSAlex Tomas blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb); 445c9de560dSAlex Tomas blocknr += first + i; 446c9de560dSAlex Tomas blocknr += 447c9de560dSAlex Tomas le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 4485d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 4495d1b1b3fSAneesh Kumar K.V __func__, "double-free of inode" 450a9df9a49STheodore Ts'o " %lu's block %llu(bit %u in group %u)", 451c9de560dSAlex Tomas inode ? inode->i_ino : 0, blocknr, 452c9de560dSAlex Tomas first + i, e4b->bd_group); 453c9de560dSAlex Tomas } 454c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 455c9de560dSAlex Tomas } 456c9de560dSAlex Tomas } 457c9de560dSAlex Tomas 458c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 459c9de560dSAlex Tomas { 460c9de560dSAlex Tomas int i; 461c9de560dSAlex Tomas 462c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 463c9de560dSAlex Tomas return; 464bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 465c9de560dSAlex Tomas for (i = 0; i < count; i++) { 466c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 467c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 468c9de560dSAlex Tomas } 469c9de560dSAlex Tomas } 470c9de560dSAlex Tomas 471c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 472c9de560dSAlex Tomas { 473c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 474c9de560dSAlex Tomas unsigned char *b1, *b2; 475c9de560dSAlex Tomas int i; 476c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 477c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 478c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 479c9de560dSAlex Tomas if (b1[i] != b2[i]) { 480a9df9a49STheodore Ts'o printk(KERN_ERR "corruption in group %u " 4814776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 4824776004fSTheodore Ts'o "on disk/prealloc\n", 483c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 484c9de560dSAlex Tomas BUG(); 485c9de560dSAlex Tomas } 486c9de560dSAlex Tomas } 487c9de560dSAlex Tomas } 488c9de560dSAlex Tomas } 489c9de560dSAlex Tomas 490c9de560dSAlex Tomas #else 491c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 492c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 493c9de560dSAlex Tomas { 494c9de560dSAlex Tomas return; 495c9de560dSAlex Tomas } 496c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 497c9de560dSAlex Tomas int first, int count) 498c9de560dSAlex Tomas { 499c9de560dSAlex Tomas return; 500c9de560dSAlex Tomas } 501c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 502c9de560dSAlex Tomas { 503c9de560dSAlex Tomas return; 504c9de560dSAlex Tomas } 505c9de560dSAlex Tomas #endif 506c9de560dSAlex Tomas 507c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 508c9de560dSAlex Tomas 509c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 510c9de560dSAlex Tomas do { \ 511c9de560dSAlex Tomas if (!(assert)) { \ 512c9de560dSAlex Tomas printk(KERN_EMERG \ 513c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 514c9de560dSAlex Tomas function, file, line, # assert); \ 515c9de560dSAlex Tomas BUG(); \ 516c9de560dSAlex Tomas } \ 517c9de560dSAlex Tomas } while (0) 518c9de560dSAlex Tomas 519c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 520c9de560dSAlex Tomas const char *function, int line) 521c9de560dSAlex Tomas { 522c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 523c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 524c9de560dSAlex Tomas int max; 525c9de560dSAlex Tomas int max2; 526c9de560dSAlex Tomas int i; 527c9de560dSAlex Tomas int j; 528c9de560dSAlex Tomas int k; 529c9de560dSAlex Tomas int count; 530c9de560dSAlex Tomas struct ext4_group_info *grp; 531c9de560dSAlex Tomas int fragments = 0; 532c9de560dSAlex Tomas int fstart; 533c9de560dSAlex Tomas struct list_head *cur; 534c9de560dSAlex Tomas void *buddy; 535c9de560dSAlex Tomas void *buddy2; 536c9de560dSAlex Tomas 537c9de560dSAlex Tomas { 538c9de560dSAlex Tomas static int mb_check_counter; 539c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 540c9de560dSAlex Tomas return 0; 541c9de560dSAlex Tomas } 542c9de560dSAlex Tomas 543c9de560dSAlex Tomas while (order > 1) { 544c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 545c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 546c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 547c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 548c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 549c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 550c9de560dSAlex Tomas 551c9de560dSAlex Tomas count = 0; 552c9de560dSAlex Tomas for (i = 0; i < max; i++) { 553c9de560dSAlex Tomas 554c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 555c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 556c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 557c9de560dSAlex Tomas MB_CHECK_ASSERT( 558c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 559c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 560c9de560dSAlex Tomas MB_CHECK_ASSERT( 561c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 562c9de560dSAlex Tomas } 563c9de560dSAlex Tomas continue; 564c9de560dSAlex Tomas } 565c9de560dSAlex Tomas 566c9de560dSAlex Tomas /* both bits in buddy2 must be 0 */ 567c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 568c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 569c9de560dSAlex Tomas 570c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 571c9de560dSAlex Tomas k = (i * (1 << order)) + j; 572c9de560dSAlex Tomas MB_CHECK_ASSERT( 573c9de560dSAlex Tomas !mb_test_bit(k, EXT4_MB_BITMAP(e4b))); 574c9de560dSAlex Tomas } 575c9de560dSAlex Tomas count++; 576c9de560dSAlex Tomas } 577c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 578c9de560dSAlex Tomas order--; 579c9de560dSAlex Tomas } 580c9de560dSAlex Tomas 581c9de560dSAlex Tomas fstart = -1; 582c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 583c9de560dSAlex Tomas for (i = 0; i < max; i++) { 584c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 585c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 586c9de560dSAlex Tomas if (fstart == -1) { 587c9de560dSAlex Tomas fragments++; 588c9de560dSAlex Tomas fstart = i; 589c9de560dSAlex Tomas } 590c9de560dSAlex Tomas continue; 591c9de560dSAlex Tomas } 592c9de560dSAlex Tomas fstart = -1; 593c9de560dSAlex Tomas /* check used bits only */ 594c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 595c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 596c9de560dSAlex Tomas k = i >> j; 597c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 598c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 599c9de560dSAlex Tomas } 600c9de560dSAlex Tomas } 601c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 602c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 603c9de560dSAlex Tomas 604c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 605c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 606c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 607c9de560dSAlex Tomas ext4_group_t groupnr; 608c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 60960bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 61060bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 611c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 61260bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 613c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 614c9de560dSAlex Tomas } 615c9de560dSAlex Tomas return 0; 616c9de560dSAlex Tomas } 617c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 618c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 61946e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 620c9de560dSAlex Tomas #else 621c9de560dSAlex Tomas #define mb_check_buddy(e4b) 622c9de560dSAlex Tomas #endif 623c9de560dSAlex Tomas 624c9de560dSAlex Tomas /* FIXME!! need more doc */ 625c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 626a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 627c9de560dSAlex Tomas struct ext4_group_info *grp) 628c9de560dSAlex Tomas { 629c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 630a36b4498SEric Sandeen ext4_grpblk_t min; 631a36b4498SEric Sandeen ext4_grpblk_t max; 632a36b4498SEric Sandeen ext4_grpblk_t chunk; 633c9de560dSAlex Tomas unsigned short border; 634c9de560dSAlex Tomas 635b73fce69SValerie Clement BUG_ON(len > EXT4_BLOCKS_PER_GROUP(sb)); 636c9de560dSAlex Tomas 637c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 638c9de560dSAlex Tomas 639c9de560dSAlex Tomas while (len > 0) { 640c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 641c9de560dSAlex Tomas max = ffs(first | border) - 1; 642c9de560dSAlex Tomas 643c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 644c9de560dSAlex Tomas min = fls(len) - 1; 645c9de560dSAlex Tomas 646c9de560dSAlex Tomas if (max < min) 647c9de560dSAlex Tomas min = max; 648c9de560dSAlex Tomas chunk = 1 << min; 649c9de560dSAlex Tomas 650c9de560dSAlex Tomas /* mark multiblock chunks only */ 651c9de560dSAlex Tomas grp->bb_counters[min]++; 652c9de560dSAlex Tomas if (min > 0) 653c9de560dSAlex Tomas mb_clear_bit(first >> min, 654c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 655c9de560dSAlex Tomas 656c9de560dSAlex Tomas len -= chunk; 657c9de560dSAlex Tomas first += chunk; 658c9de560dSAlex Tomas } 659c9de560dSAlex Tomas } 660c9de560dSAlex Tomas 661089ceeccSEric Sandeen static noinline_for_stack 662089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 663c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 664c9de560dSAlex Tomas { 665c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 666a36b4498SEric Sandeen ext4_grpblk_t max = EXT4_BLOCKS_PER_GROUP(sb); 667a36b4498SEric Sandeen ext4_grpblk_t i = 0; 668a36b4498SEric Sandeen ext4_grpblk_t first; 669a36b4498SEric Sandeen ext4_grpblk_t len; 670c9de560dSAlex Tomas unsigned free = 0; 671c9de560dSAlex Tomas unsigned fragments = 0; 672c9de560dSAlex Tomas unsigned long long period = get_cycles(); 673c9de560dSAlex Tomas 674c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 675c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 676ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 677c9de560dSAlex Tomas grp->bb_first_free = i; 678c9de560dSAlex Tomas while (i < max) { 679c9de560dSAlex Tomas fragments++; 680c9de560dSAlex Tomas first = i; 681ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 682c9de560dSAlex Tomas len = i - first; 683c9de560dSAlex Tomas free += len; 684c9de560dSAlex Tomas if (len > 1) 685c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 686c9de560dSAlex Tomas else 687c9de560dSAlex Tomas grp->bb_counters[0]++; 688c9de560dSAlex Tomas if (i < max) 689ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 690c9de560dSAlex Tomas } 691c9de560dSAlex Tomas grp->bb_fragments = fragments; 692c9de560dSAlex Tomas 693c9de560dSAlex Tomas if (free != grp->bb_free) { 6945d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, group, __func__, 695a9df9a49STheodore Ts'o "EXT4-fs: group %u: %u blocks in bitmap, %u in gd", 696c9de560dSAlex Tomas group, free, grp->bb_free); 697e56eb659SAneesh Kumar K.V /* 698e56eb659SAneesh Kumar K.V * If we intent to continue, we consider group descritor 699e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 700e56eb659SAneesh Kumar K.V */ 701c9de560dSAlex Tomas grp->bb_free = free; 702c9de560dSAlex Tomas } 703c9de560dSAlex Tomas 704c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 705c9de560dSAlex Tomas 706c9de560dSAlex Tomas period = get_cycles() - period; 707c9de560dSAlex Tomas spin_lock(&EXT4_SB(sb)->s_bal_lock); 708c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_buddies_generated++; 709c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_generation_time += period; 710c9de560dSAlex Tomas spin_unlock(&EXT4_SB(sb)->s_bal_lock); 711c9de560dSAlex Tomas } 712c9de560dSAlex Tomas 713c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 714c9de560dSAlex Tomas * for convenience. The information regarding each group 715c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 716c9de560dSAlex Tomas * block bitmap and buddy information. The information are 717c9de560dSAlex Tomas * stored in the inode as 718c9de560dSAlex Tomas * 719c9de560dSAlex Tomas * { page } 720c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 721c9de560dSAlex Tomas * 722c9de560dSAlex Tomas * 723c9de560dSAlex Tomas * one block each for bitmap and buddy information. 724c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 725c9de560dSAlex Tomas * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks. 726c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 727c9de560dSAlex Tomas * is blocks_per_page/2 728c9de560dSAlex Tomas */ 729c9de560dSAlex Tomas 730c9de560dSAlex Tomas static int ext4_mb_init_cache(struct page *page, char *incore) 731c9de560dSAlex Tomas { 7328df9675fSTheodore Ts'o ext4_group_t ngroups; 733c9de560dSAlex Tomas int blocksize; 734c9de560dSAlex Tomas int blocks_per_page; 735c9de560dSAlex Tomas int groups_per_page; 736c9de560dSAlex Tomas int err = 0; 737c9de560dSAlex Tomas int i; 738c9de560dSAlex Tomas ext4_group_t first_group; 739c9de560dSAlex Tomas int first_block; 740c9de560dSAlex Tomas struct super_block *sb; 741c9de560dSAlex Tomas struct buffer_head *bhs; 742c9de560dSAlex Tomas struct buffer_head **bh; 743c9de560dSAlex Tomas struct inode *inode; 744c9de560dSAlex Tomas char *data; 745c9de560dSAlex Tomas char *bitmap; 746c9de560dSAlex Tomas 7476ba495e9STheodore Ts'o mb_debug(1, "init page %lu\n", page->index); 748c9de560dSAlex Tomas 749c9de560dSAlex Tomas inode = page->mapping->host; 750c9de560dSAlex Tomas sb = inode->i_sb; 7518df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 752c9de560dSAlex Tomas blocksize = 1 << inode->i_blkbits; 753c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / blocksize; 754c9de560dSAlex Tomas 755c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 756c9de560dSAlex Tomas if (groups_per_page == 0) 757c9de560dSAlex Tomas groups_per_page = 1; 758c9de560dSAlex Tomas 759c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 760c9de560dSAlex Tomas if (groups_per_page > 1) { 761c9de560dSAlex Tomas err = -ENOMEM; 762c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 763c9de560dSAlex Tomas bh = kzalloc(i, GFP_NOFS); 764c9de560dSAlex Tomas if (bh == NULL) 765c9de560dSAlex Tomas goto out; 766c9de560dSAlex Tomas } else 767c9de560dSAlex Tomas bh = &bhs; 768c9de560dSAlex Tomas 769c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 770c9de560dSAlex Tomas 771c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 772c9de560dSAlex Tomas for (i = 0; i < groups_per_page; i++) { 773c9de560dSAlex Tomas struct ext4_group_desc *desc; 774c9de560dSAlex Tomas 7758df9675fSTheodore Ts'o if (first_group + i >= ngroups) 776c9de560dSAlex Tomas break; 777c9de560dSAlex Tomas 778c9de560dSAlex Tomas err = -EIO; 779c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, first_group + i, NULL); 780c9de560dSAlex Tomas if (desc == NULL) 781c9de560dSAlex Tomas goto out; 782c9de560dSAlex Tomas 783c9de560dSAlex Tomas err = -ENOMEM; 784c9de560dSAlex Tomas bh[i] = sb_getblk(sb, ext4_block_bitmap(sb, desc)); 785c9de560dSAlex Tomas if (bh[i] == NULL) 786c9de560dSAlex Tomas goto out; 787c9de560dSAlex Tomas 7882ccb5fb9SAneesh Kumar K.V if (bitmap_uptodate(bh[i])) 789c9de560dSAlex Tomas continue; 790c9de560dSAlex Tomas 791c806e68fSFrederic Bohe lock_buffer(bh[i]); 7922ccb5fb9SAneesh Kumar K.V if (bitmap_uptodate(bh[i])) { 7932ccb5fb9SAneesh Kumar K.V unlock_buffer(bh[i]); 7942ccb5fb9SAneesh Kumar K.V continue; 7952ccb5fb9SAneesh Kumar K.V } 796955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, first_group + i); 797c9de560dSAlex Tomas if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 798c9de560dSAlex Tomas ext4_init_block_bitmap(sb, bh[i], 799c9de560dSAlex Tomas first_group + i, desc); 8002ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 801c9de560dSAlex Tomas set_buffer_uptodate(bh[i]); 802955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, first_group + i); 8033300bedaSAneesh Kumar K.V unlock_buffer(bh[i]); 804c9de560dSAlex Tomas continue; 805c9de560dSAlex Tomas } 806955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, first_group + i); 8072ccb5fb9SAneesh Kumar K.V if (buffer_uptodate(bh[i])) { 8082ccb5fb9SAneesh Kumar K.V /* 8092ccb5fb9SAneesh Kumar K.V * if not uninit if bh is uptodate, 8102ccb5fb9SAneesh Kumar K.V * bitmap is also uptodate 8112ccb5fb9SAneesh Kumar K.V */ 8122ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 8132ccb5fb9SAneesh Kumar K.V unlock_buffer(bh[i]); 8142ccb5fb9SAneesh Kumar K.V continue; 8152ccb5fb9SAneesh Kumar K.V } 816c9de560dSAlex Tomas get_bh(bh[i]); 8172ccb5fb9SAneesh Kumar K.V /* 8182ccb5fb9SAneesh Kumar K.V * submit the buffer_head for read. We can 8192ccb5fb9SAneesh Kumar K.V * safely mark the bitmap as uptodate now. 8202ccb5fb9SAneesh Kumar K.V * We do it here so the bitmap uptodate bit 8212ccb5fb9SAneesh Kumar K.V * get set with buffer lock held. 8222ccb5fb9SAneesh Kumar K.V */ 8232ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 824c9de560dSAlex Tomas bh[i]->b_end_io = end_buffer_read_sync; 825c9de560dSAlex Tomas submit_bh(READ, bh[i]); 8266ba495e9STheodore Ts'o mb_debug(1, "read bitmap for group %u\n", first_group + i); 827c9de560dSAlex Tomas } 828c9de560dSAlex Tomas 829c9de560dSAlex Tomas /* wait for I/O completion */ 830c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 831c9de560dSAlex Tomas wait_on_buffer(bh[i]); 832c9de560dSAlex Tomas 833c9de560dSAlex Tomas err = -EIO; 834c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 835c9de560dSAlex Tomas if (!buffer_uptodate(bh[i])) 836c9de560dSAlex Tomas goto out; 837c9de560dSAlex Tomas 83831b481dcSMingming Cao err = 0; 839c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 84029eaf024SAneesh Kumar K.V /* init the page */ 84129eaf024SAneesh Kumar K.V memset(page_address(page), 0xff, PAGE_CACHE_SIZE); 842c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 843c9de560dSAlex Tomas int group; 844c9de560dSAlex Tomas struct ext4_group_info *grinfo; 845c9de560dSAlex Tomas 846c9de560dSAlex Tomas group = (first_block + i) >> 1; 8478df9675fSTheodore Ts'o if (group >= ngroups) 848c9de560dSAlex Tomas break; 849c9de560dSAlex Tomas 850c9de560dSAlex Tomas /* 851c9de560dSAlex Tomas * data carry information regarding this 852c9de560dSAlex Tomas * particular group in the format specified 853c9de560dSAlex Tomas * above 854c9de560dSAlex Tomas * 855c9de560dSAlex Tomas */ 856c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 857c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 858c9de560dSAlex Tomas 859c9de560dSAlex Tomas /* 860c9de560dSAlex Tomas * We place the buddy block and bitmap block 861c9de560dSAlex Tomas * close together 862c9de560dSAlex Tomas */ 863c9de560dSAlex Tomas if ((first_block + i) & 1) { 864c9de560dSAlex Tomas /* this is block of buddy */ 865c9de560dSAlex Tomas BUG_ON(incore == NULL); 8666ba495e9STheodore Ts'o mb_debug(1, "put buddy for group %u in page %lu/%x\n", 867c9de560dSAlex Tomas group, page->index, i * blocksize); 868c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 869c9de560dSAlex Tomas grinfo->bb_fragments = 0; 870c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 8711927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 8721927805eSEric Sandeen (sb->s_blocksize_bits+2)); 873c9de560dSAlex Tomas /* 874c9de560dSAlex Tomas * incore got set to the group block bitmap below 875c9de560dSAlex Tomas */ 8767a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 877c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 8787a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 879c9de560dSAlex Tomas incore = NULL; 880c9de560dSAlex Tomas } else { 881c9de560dSAlex Tomas /* this is block of bitmap */ 882c9de560dSAlex Tomas BUG_ON(incore != NULL); 8836ba495e9STheodore Ts'o mb_debug(1, "put bitmap for group %u in page %lu/%x\n", 884c9de560dSAlex Tomas group, page->index, i * blocksize); 885c9de560dSAlex Tomas 886c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 887c9de560dSAlex Tomas ext4_lock_group(sb, group); 888c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 889c9de560dSAlex Tomas 890c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 891c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 8927a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 893c9de560dSAlex Tomas ext4_unlock_group(sb, group); 894c9de560dSAlex Tomas 895c9de560dSAlex Tomas /* set incore so that the buddy information can be 896c9de560dSAlex Tomas * generated using this 897c9de560dSAlex Tomas */ 898c9de560dSAlex Tomas incore = data; 899c9de560dSAlex Tomas } 900c9de560dSAlex Tomas } 901c9de560dSAlex Tomas SetPageUptodate(page); 902c9de560dSAlex Tomas 903c9de560dSAlex Tomas out: 904c9de560dSAlex Tomas if (bh) { 905c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 906c9de560dSAlex Tomas brelse(bh[i]); 907c9de560dSAlex Tomas if (bh != &bhs) 908c9de560dSAlex Tomas kfree(bh); 909c9de560dSAlex Tomas } 910c9de560dSAlex Tomas return err; 911c9de560dSAlex Tomas } 912c9de560dSAlex Tomas 913b6a758ecSAneesh Kumar K.V static noinline_for_stack 914b6a758ecSAneesh Kumar K.V int ext4_mb_init_group(struct super_block *sb, ext4_group_t group) 915b6a758ecSAneesh Kumar K.V { 916b6a758ecSAneesh Kumar K.V 917b6a758ecSAneesh Kumar K.V int ret = 0; 918b6a758ecSAneesh Kumar K.V void *bitmap; 919b6a758ecSAneesh Kumar K.V int blocks_per_page; 920b6a758ecSAneesh Kumar K.V int block, pnum, poff; 921b6a758ecSAneesh Kumar K.V int num_grp_locked = 0; 922b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 923b6a758ecSAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 924b6a758ecSAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 925b6a758ecSAneesh Kumar K.V struct page *page = NULL, *bitmap_page = NULL; 926b6a758ecSAneesh Kumar K.V 927b6a758ecSAneesh Kumar K.V mb_debug(1, "init group %u\n", group); 928b6a758ecSAneesh Kumar K.V blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 929b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 930b6a758ecSAneesh Kumar K.V /* 93108c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 93208c3a813SAneesh Kumar K.V * page which map to the group from which we are already 93308c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 93408c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 93508c3a813SAneesh Kumar K.V * would have taken the alloc_sem lock. 936b6a758ecSAneesh Kumar K.V */ 937b6a758ecSAneesh Kumar K.V num_grp_locked = ext4_mb_get_buddy_cache_lock(sb, group); 938b6a758ecSAneesh Kumar K.V if (!EXT4_MB_GRP_NEED_INIT(this_grp)) { 939b6a758ecSAneesh Kumar K.V /* 940b6a758ecSAneesh Kumar K.V * somebody initialized the group 941b6a758ecSAneesh Kumar K.V * return without doing anything 942b6a758ecSAneesh Kumar K.V */ 943b6a758ecSAneesh Kumar K.V ret = 0; 944b6a758ecSAneesh Kumar K.V goto err; 945b6a758ecSAneesh Kumar K.V } 946b6a758ecSAneesh Kumar K.V /* 947b6a758ecSAneesh Kumar K.V * the buddy cache inode stores the block bitmap 948b6a758ecSAneesh Kumar K.V * and buddy information in consecutive blocks. 949b6a758ecSAneesh Kumar K.V * So for each group we need two blocks. 950b6a758ecSAneesh Kumar K.V */ 951b6a758ecSAneesh Kumar K.V block = group * 2; 952b6a758ecSAneesh Kumar K.V pnum = block / blocks_per_page; 953b6a758ecSAneesh Kumar K.V poff = block % blocks_per_page; 954b6a758ecSAneesh Kumar K.V page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 955b6a758ecSAneesh Kumar K.V if (page) { 956b6a758ecSAneesh Kumar K.V BUG_ON(page->mapping != inode->i_mapping); 957b6a758ecSAneesh Kumar K.V ret = ext4_mb_init_cache(page, NULL); 958b6a758ecSAneesh Kumar K.V if (ret) { 959b6a758ecSAneesh Kumar K.V unlock_page(page); 960b6a758ecSAneesh Kumar K.V goto err; 961b6a758ecSAneesh Kumar K.V } 962b6a758ecSAneesh Kumar K.V unlock_page(page); 963b6a758ecSAneesh Kumar K.V } 964b6a758ecSAneesh Kumar K.V if (page == NULL || !PageUptodate(page)) { 965b6a758ecSAneesh Kumar K.V ret = -EIO; 966b6a758ecSAneesh Kumar K.V goto err; 967b6a758ecSAneesh Kumar K.V } 968b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 969b6a758ecSAneesh Kumar K.V bitmap_page = page; 970b6a758ecSAneesh Kumar K.V bitmap = page_address(page) + (poff * sb->s_blocksize); 971b6a758ecSAneesh Kumar K.V 972b6a758ecSAneesh Kumar K.V /* init buddy cache */ 973b6a758ecSAneesh Kumar K.V block++; 974b6a758ecSAneesh Kumar K.V pnum = block / blocks_per_page; 975b6a758ecSAneesh Kumar K.V poff = block % blocks_per_page; 976b6a758ecSAneesh Kumar K.V page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 977b6a758ecSAneesh Kumar K.V if (page == bitmap_page) { 978b6a758ecSAneesh Kumar K.V /* 979b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 980b6a758ecSAneesh Kumar K.V * the same page we don't need to force 981b6a758ecSAneesh Kumar K.V * init the buddy 982b6a758ecSAneesh Kumar K.V */ 983b6a758ecSAneesh Kumar K.V unlock_page(page); 984b6a758ecSAneesh Kumar K.V } else if (page) { 985b6a758ecSAneesh Kumar K.V BUG_ON(page->mapping != inode->i_mapping); 986b6a758ecSAneesh Kumar K.V ret = ext4_mb_init_cache(page, bitmap); 987b6a758ecSAneesh Kumar K.V if (ret) { 988b6a758ecSAneesh Kumar K.V unlock_page(page); 989b6a758ecSAneesh Kumar K.V goto err; 990b6a758ecSAneesh Kumar K.V } 991b6a758ecSAneesh Kumar K.V unlock_page(page); 992b6a758ecSAneesh Kumar K.V } 993b6a758ecSAneesh Kumar K.V if (page == NULL || !PageUptodate(page)) { 994b6a758ecSAneesh Kumar K.V ret = -EIO; 995b6a758ecSAneesh Kumar K.V goto err; 996b6a758ecSAneesh Kumar K.V } 997b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 998b6a758ecSAneesh Kumar K.V err: 999b6a758ecSAneesh Kumar K.V ext4_mb_put_buddy_cache_lock(sb, group, num_grp_locked); 1000b6a758ecSAneesh Kumar K.V if (bitmap_page) 1001b6a758ecSAneesh Kumar K.V page_cache_release(bitmap_page); 1002b6a758ecSAneesh Kumar K.V if (page) 1003b6a758ecSAneesh Kumar K.V page_cache_release(page); 1004b6a758ecSAneesh Kumar K.V return ret; 1005b6a758ecSAneesh Kumar K.V } 1006b6a758ecSAneesh Kumar K.V 10074ddfef7bSEric Sandeen static noinline_for_stack int 10084ddfef7bSEric Sandeen ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1009c9de560dSAlex Tomas struct ext4_buddy *e4b) 1010c9de560dSAlex Tomas { 1011c9de560dSAlex Tomas int blocks_per_page; 1012c9de560dSAlex Tomas int block; 1013c9de560dSAlex Tomas int pnum; 1014c9de560dSAlex Tomas int poff; 1015c9de560dSAlex Tomas struct page *page; 1016fdf6c7a7SShen Feng int ret; 1017920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1018920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1019920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1020c9de560dSAlex Tomas 10216ba495e9STheodore Ts'o mb_debug(1, "load group %u\n", group); 1022c9de560dSAlex Tomas 1023c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1024920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1025c9de560dSAlex Tomas 1026c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1027c9de560dSAlex Tomas e4b->bd_info = ext4_get_group_info(sb, group); 1028c9de560dSAlex Tomas e4b->bd_sb = sb; 1029c9de560dSAlex Tomas e4b->bd_group = group; 1030c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1031c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1032920313a7SAneesh Kumar K.V e4b->alloc_semp = &grp->alloc_sem; 1033920313a7SAneesh Kumar K.V 1034920313a7SAneesh Kumar K.V /* Take the read lock on the group alloc 1035920313a7SAneesh Kumar K.V * sem. This would make sure a parallel 1036920313a7SAneesh Kumar K.V * ext4_mb_init_group happening on other 1037920313a7SAneesh Kumar K.V * groups mapped by the page is blocked 1038920313a7SAneesh Kumar K.V * till we are done with allocation 1039920313a7SAneesh Kumar K.V */ 1040f41c0750SAneesh Kumar K.V repeat_load_buddy: 1041920313a7SAneesh Kumar K.V down_read(e4b->alloc_semp); 1042c9de560dSAlex Tomas 1043f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1044f41c0750SAneesh Kumar K.V /* we need to check for group need init flag 1045f41c0750SAneesh Kumar K.V * with alloc_semp held so that we can be sure 1046f41c0750SAneesh Kumar K.V * that new blocks didn't get added to the group 1047f41c0750SAneesh Kumar K.V * when we are loading the buddy cache 1048f41c0750SAneesh Kumar K.V */ 1049f41c0750SAneesh Kumar K.V up_read(e4b->alloc_semp); 1050f41c0750SAneesh Kumar K.V /* 1051f41c0750SAneesh Kumar K.V * we need full data about the group 1052f41c0750SAneesh Kumar K.V * to make a good selection 1053f41c0750SAneesh Kumar K.V */ 1054f41c0750SAneesh Kumar K.V ret = ext4_mb_init_group(sb, group); 1055f41c0750SAneesh Kumar K.V if (ret) 1056f41c0750SAneesh Kumar K.V return ret; 1057f41c0750SAneesh Kumar K.V goto repeat_load_buddy; 1058f41c0750SAneesh Kumar K.V } 1059f41c0750SAneesh Kumar K.V 1060c9de560dSAlex Tomas /* 1061c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1062c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1063c9de560dSAlex Tomas * So for each group we need two blocks. 1064c9de560dSAlex Tomas */ 1065c9de560dSAlex Tomas block = group * 2; 1066c9de560dSAlex Tomas pnum = block / blocks_per_page; 1067c9de560dSAlex Tomas poff = block % blocks_per_page; 1068c9de560dSAlex Tomas 1069c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1070c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 1071c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1072c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1073c9de560dSAlex Tomas if (page) 1074920313a7SAneesh Kumar K.V /* 1075920313a7SAneesh Kumar K.V * drop the page reference and try 1076920313a7SAneesh Kumar K.V * to get the page with lock. If we 1077920313a7SAneesh Kumar K.V * are not uptodate that implies 1078920313a7SAneesh Kumar K.V * somebody just created the page but 1079920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1080920313a7SAneesh Kumar K.V * wait for it to initialize. 1081920313a7SAneesh Kumar K.V */ 1082c9de560dSAlex Tomas page_cache_release(page); 1083c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1084c9de560dSAlex Tomas if (page) { 1085c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1086c9de560dSAlex Tomas if (!PageUptodate(page)) { 1087fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, NULL); 1088fdf6c7a7SShen Feng if (ret) { 1089fdf6c7a7SShen Feng unlock_page(page); 1090fdf6c7a7SShen Feng goto err; 1091fdf6c7a7SShen Feng } 1092c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1093c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1094c9de560dSAlex Tomas } 1095c9de560dSAlex Tomas unlock_page(page); 1096c9de560dSAlex Tomas } 1097c9de560dSAlex Tomas } 1098fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1099fdf6c7a7SShen Feng ret = -EIO; 1100c9de560dSAlex Tomas goto err; 1101fdf6c7a7SShen Feng } 1102c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1103c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1104c9de560dSAlex Tomas mark_page_accessed(page); 1105c9de560dSAlex Tomas 1106c9de560dSAlex Tomas block++; 1107c9de560dSAlex Tomas pnum = block / blocks_per_page; 1108c9de560dSAlex Tomas poff = block % blocks_per_page; 1109c9de560dSAlex Tomas 1110c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1111c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1112c9de560dSAlex Tomas if (page) 1113c9de560dSAlex Tomas page_cache_release(page); 1114c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1115c9de560dSAlex Tomas if (page) { 1116c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1117fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1118fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, e4b->bd_bitmap); 1119fdf6c7a7SShen Feng if (ret) { 1120fdf6c7a7SShen Feng unlock_page(page); 1121fdf6c7a7SShen Feng goto err; 1122fdf6c7a7SShen Feng } 1123fdf6c7a7SShen Feng } 1124c9de560dSAlex Tomas unlock_page(page); 1125c9de560dSAlex Tomas } 1126c9de560dSAlex Tomas } 1127fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1128fdf6c7a7SShen Feng ret = -EIO; 1129c9de560dSAlex Tomas goto err; 1130fdf6c7a7SShen Feng } 1131c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1132c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1133c9de560dSAlex Tomas mark_page_accessed(page); 1134c9de560dSAlex Tomas 1135c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 1136c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 1137c9de560dSAlex Tomas 1138c9de560dSAlex Tomas return 0; 1139c9de560dSAlex Tomas 1140c9de560dSAlex Tomas err: 1141c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1142c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1143c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1144c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1145c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1146c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1147920313a7SAneesh Kumar K.V 1148920313a7SAneesh Kumar K.V /* Done with the buddy cache */ 1149920313a7SAneesh Kumar K.V up_read(e4b->alloc_semp); 1150fdf6c7a7SShen Feng return ret; 1151c9de560dSAlex Tomas } 1152c9de560dSAlex Tomas 1153c9de560dSAlex Tomas static void ext4_mb_release_desc(struct ext4_buddy *e4b) 1154c9de560dSAlex Tomas { 1155c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1156c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1157c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1158c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1159920313a7SAneesh Kumar K.V /* Done with the buddy cache */ 11608556e8f3SAneesh Kumar K.V if (e4b->alloc_semp) 1161920313a7SAneesh Kumar K.V up_read(e4b->alloc_semp); 1162c9de560dSAlex Tomas } 1163c9de560dSAlex Tomas 1164c9de560dSAlex Tomas 1165c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1166c9de560dSAlex Tomas { 1167c9de560dSAlex Tomas int order = 1; 1168c9de560dSAlex Tomas void *bb; 1169c9de560dSAlex Tomas 1170c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 1171c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1172c9de560dSAlex Tomas 1173c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b); 1174c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1175c9de560dSAlex Tomas block = block >> 1; 1176c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1177c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1178c9de560dSAlex Tomas return order; 1179c9de560dSAlex Tomas } 1180c9de560dSAlex Tomas bb += 1 << (e4b->bd_blkbits - order); 1181c9de560dSAlex Tomas order++; 1182c9de560dSAlex Tomas } 1183c9de560dSAlex Tomas return 0; 1184c9de560dSAlex Tomas } 1185c9de560dSAlex Tomas 1186955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1187c9de560dSAlex Tomas { 1188c9de560dSAlex Tomas __u32 *addr; 1189c9de560dSAlex Tomas 1190c9de560dSAlex Tomas len = cur + len; 1191c9de560dSAlex Tomas while (cur < len) { 1192c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1193c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1194c9de560dSAlex Tomas addr = bm + (cur >> 3); 1195c9de560dSAlex Tomas *addr = 0; 1196c9de560dSAlex Tomas cur += 32; 1197c9de560dSAlex Tomas continue; 1198c9de560dSAlex Tomas } 1199e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1200c9de560dSAlex Tomas cur++; 1201c9de560dSAlex Tomas } 1202c9de560dSAlex Tomas } 1203c9de560dSAlex Tomas 1204955ce5f5SAneesh Kumar K.V static void mb_set_bits(void *bm, int cur, int len) 1205c9de560dSAlex Tomas { 1206c9de560dSAlex Tomas __u32 *addr; 1207c9de560dSAlex Tomas 1208c9de560dSAlex Tomas len = cur + len; 1209c9de560dSAlex Tomas while (cur < len) { 1210c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1211c9de560dSAlex Tomas /* fast path: set whole word at once */ 1212c9de560dSAlex Tomas addr = bm + (cur >> 3); 1213c9de560dSAlex Tomas *addr = 0xffffffff; 1214c9de560dSAlex Tomas cur += 32; 1215c9de560dSAlex Tomas continue; 1216c9de560dSAlex Tomas } 1217e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1218c9de560dSAlex Tomas cur++; 1219c9de560dSAlex Tomas } 1220c9de560dSAlex Tomas } 1221c9de560dSAlex Tomas 12227e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1223c9de560dSAlex Tomas int first, int count) 1224c9de560dSAlex Tomas { 1225c9de560dSAlex Tomas int block = 0; 1226c9de560dSAlex Tomas int max = 0; 1227c9de560dSAlex Tomas int order; 1228c9de560dSAlex Tomas void *buddy; 1229c9de560dSAlex Tomas void *buddy2; 1230c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1231c9de560dSAlex Tomas 1232c9de560dSAlex Tomas BUG_ON(first + count > (sb->s_blocksize << 3)); 1233bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1234c9de560dSAlex Tomas mb_check_buddy(e4b); 1235c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1236c9de560dSAlex Tomas 1237c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1238c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1239c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1240c9de560dSAlex Tomas 1241c9de560dSAlex Tomas /* let's maintain fragments counter */ 1242c9de560dSAlex Tomas if (first != 0) 1243c9de560dSAlex Tomas block = !mb_test_bit(first - 1, EXT4_MB_BITMAP(e4b)); 1244c9de560dSAlex Tomas if (first + count < EXT4_SB(sb)->s_mb_maxs[0]) 1245c9de560dSAlex Tomas max = !mb_test_bit(first + count, EXT4_MB_BITMAP(e4b)); 1246c9de560dSAlex Tomas if (block && max) 1247c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1248c9de560dSAlex Tomas else if (!block && !max) 1249c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1250c9de560dSAlex Tomas 1251c9de560dSAlex Tomas /* let's maintain buddy itself */ 1252c9de560dSAlex Tomas while (count-- > 0) { 1253c9de560dSAlex Tomas block = first++; 1254c9de560dSAlex Tomas order = 0; 1255c9de560dSAlex Tomas 1256c9de560dSAlex Tomas if (!mb_test_bit(block, EXT4_MB_BITMAP(e4b))) { 1257c9de560dSAlex Tomas ext4_fsblk_t blocknr; 1258c9de560dSAlex Tomas blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb); 1259c9de560dSAlex Tomas blocknr += block; 1260c9de560dSAlex Tomas blocknr += 1261c9de560dSAlex Tomas le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 12625d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 12635d1b1b3fSAneesh Kumar K.V __func__, "double-free of inode" 1264a9df9a49STheodore Ts'o " %lu's block %llu(bit %u in group %u)", 1265c9de560dSAlex Tomas inode ? inode->i_ino : 0, blocknr, block, 1266c9de560dSAlex Tomas e4b->bd_group); 1267c9de560dSAlex Tomas } 1268c9de560dSAlex Tomas mb_clear_bit(block, EXT4_MB_BITMAP(e4b)); 1269c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1270c9de560dSAlex Tomas 1271c9de560dSAlex Tomas /* start of the buddy */ 1272c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1273c9de560dSAlex Tomas 1274c9de560dSAlex Tomas do { 1275c9de560dSAlex Tomas block &= ~1UL; 1276c9de560dSAlex Tomas if (mb_test_bit(block, buddy) || 1277c9de560dSAlex Tomas mb_test_bit(block + 1, buddy)) 1278c9de560dSAlex Tomas break; 1279c9de560dSAlex Tomas 1280c9de560dSAlex Tomas /* both the buddies are free, try to coalesce them */ 1281c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order + 1, &max); 1282c9de560dSAlex Tomas 1283c9de560dSAlex Tomas if (!buddy2) 1284c9de560dSAlex Tomas break; 1285c9de560dSAlex Tomas 1286c9de560dSAlex Tomas if (order > 0) { 1287c9de560dSAlex Tomas /* for special purposes, we don't set 1288c9de560dSAlex Tomas * free bits in bitmap */ 1289c9de560dSAlex Tomas mb_set_bit(block, buddy); 1290c9de560dSAlex Tomas mb_set_bit(block + 1, buddy); 1291c9de560dSAlex Tomas } 1292c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1293c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1294c9de560dSAlex Tomas 1295c9de560dSAlex Tomas block = block >> 1; 1296c9de560dSAlex Tomas order++; 1297c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1298c9de560dSAlex Tomas 1299c9de560dSAlex Tomas mb_clear_bit(block, buddy2); 1300c9de560dSAlex Tomas buddy = buddy2; 1301c9de560dSAlex Tomas } while (1); 1302c9de560dSAlex Tomas } 1303c9de560dSAlex Tomas mb_check_buddy(e4b); 1304c9de560dSAlex Tomas } 1305c9de560dSAlex Tomas 1306c9de560dSAlex Tomas static int mb_find_extent(struct ext4_buddy *e4b, int order, int block, 1307c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1308c9de560dSAlex Tomas { 1309c9de560dSAlex Tomas int next = block; 1310c9de560dSAlex Tomas int max; 1311c9de560dSAlex Tomas int ord; 1312c9de560dSAlex Tomas void *buddy; 1313c9de560dSAlex Tomas 1314bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1315c9de560dSAlex Tomas BUG_ON(ex == NULL); 1316c9de560dSAlex Tomas 1317c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1318c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1319c9de560dSAlex Tomas BUG_ON(block >= max); 1320c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1321c9de560dSAlex Tomas ex->fe_len = 0; 1322c9de560dSAlex Tomas ex->fe_start = 0; 1323c9de560dSAlex Tomas ex->fe_group = 0; 1324c9de560dSAlex Tomas return 0; 1325c9de560dSAlex Tomas } 1326c9de560dSAlex Tomas 1327c9de560dSAlex Tomas /* FIXME dorp order completely ? */ 1328c9de560dSAlex Tomas if (likely(order == 0)) { 1329c9de560dSAlex Tomas /* find actual order */ 1330c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1331c9de560dSAlex Tomas block = block >> order; 1332c9de560dSAlex Tomas } 1333c9de560dSAlex Tomas 1334c9de560dSAlex Tomas ex->fe_len = 1 << order; 1335c9de560dSAlex Tomas ex->fe_start = block << order; 1336c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1337c9de560dSAlex Tomas 1338c9de560dSAlex Tomas /* calc difference from given start */ 1339c9de560dSAlex Tomas next = next - ex->fe_start; 1340c9de560dSAlex Tomas ex->fe_len -= next; 1341c9de560dSAlex Tomas ex->fe_start += next; 1342c9de560dSAlex Tomas 1343c9de560dSAlex Tomas while (needed > ex->fe_len && 1344c9de560dSAlex Tomas (buddy = mb_find_buddy(e4b, order, &max))) { 1345c9de560dSAlex Tomas 1346c9de560dSAlex Tomas if (block + 1 >= max) 1347c9de560dSAlex Tomas break; 1348c9de560dSAlex Tomas 1349c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1350c9de560dSAlex Tomas if (mb_test_bit(next, EXT4_MB_BITMAP(e4b))) 1351c9de560dSAlex Tomas break; 1352c9de560dSAlex Tomas 1353c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, next); 1354c9de560dSAlex Tomas 1355c9de560dSAlex Tomas order = ord; 1356c9de560dSAlex Tomas block = next >> order; 1357c9de560dSAlex Tomas ex->fe_len += 1 << order; 1358c9de560dSAlex Tomas } 1359c9de560dSAlex Tomas 1360c9de560dSAlex Tomas BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3))); 1361c9de560dSAlex Tomas return ex->fe_len; 1362c9de560dSAlex Tomas } 1363c9de560dSAlex Tomas 1364c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1365c9de560dSAlex Tomas { 1366c9de560dSAlex Tomas int ord; 1367c9de560dSAlex Tomas int mlen = 0; 1368c9de560dSAlex Tomas int max = 0; 1369c9de560dSAlex Tomas int cur; 1370c9de560dSAlex Tomas int start = ex->fe_start; 1371c9de560dSAlex Tomas int len = ex->fe_len; 1372c9de560dSAlex Tomas unsigned ret = 0; 1373c9de560dSAlex Tomas int len0 = len; 1374c9de560dSAlex Tomas void *buddy; 1375c9de560dSAlex Tomas 1376c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1377c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1378bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1379c9de560dSAlex Tomas mb_check_buddy(e4b); 1380c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1381c9de560dSAlex Tomas 1382c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1383c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1384c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1385c9de560dSAlex Tomas 1386c9de560dSAlex Tomas /* let's maintain fragments counter */ 1387c9de560dSAlex Tomas if (start != 0) 1388c9de560dSAlex Tomas mlen = !mb_test_bit(start - 1, EXT4_MB_BITMAP(e4b)); 1389c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1390c9de560dSAlex Tomas max = !mb_test_bit(start + len, EXT4_MB_BITMAP(e4b)); 1391c9de560dSAlex Tomas if (mlen && max) 1392c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1393c9de560dSAlex Tomas else if (!mlen && !max) 1394c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1395c9de560dSAlex Tomas 1396c9de560dSAlex Tomas /* let's maintain buddy itself */ 1397c9de560dSAlex Tomas while (len) { 1398c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1399c9de560dSAlex Tomas 1400c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1401c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1402c9de560dSAlex Tomas mlen = 1 << ord; 1403c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1404c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1405c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1406c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1407c9de560dSAlex Tomas start += mlen; 1408c9de560dSAlex Tomas len -= mlen; 1409c9de560dSAlex Tomas BUG_ON(len < 0); 1410c9de560dSAlex Tomas continue; 1411c9de560dSAlex Tomas } 1412c9de560dSAlex Tomas 1413c9de560dSAlex Tomas /* store for history */ 1414c9de560dSAlex Tomas if (ret == 0) 1415c9de560dSAlex Tomas ret = len | (ord << 16); 1416c9de560dSAlex Tomas 1417c9de560dSAlex Tomas /* we have to split large buddy */ 1418c9de560dSAlex Tomas BUG_ON(ord <= 0); 1419c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1420c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1421c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1422c9de560dSAlex Tomas 1423c9de560dSAlex Tomas ord--; 1424c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1425c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1426c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1427c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1428c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1429c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1430c9de560dSAlex Tomas } 1431c9de560dSAlex Tomas 1432955ce5f5SAneesh Kumar K.V mb_set_bits(EXT4_MB_BITMAP(e4b), ex->fe_start, len0); 1433c9de560dSAlex Tomas mb_check_buddy(e4b); 1434c9de560dSAlex Tomas 1435c9de560dSAlex Tomas return ret; 1436c9de560dSAlex Tomas } 1437c9de560dSAlex Tomas 1438c9de560dSAlex Tomas /* 1439c9de560dSAlex Tomas * Must be called under group lock! 1440c9de560dSAlex Tomas */ 1441c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1442c9de560dSAlex Tomas struct ext4_buddy *e4b) 1443c9de560dSAlex Tomas { 1444c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1445c9de560dSAlex Tomas int ret; 1446c9de560dSAlex Tomas 1447c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1448c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1449c9de560dSAlex Tomas 1450c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1451c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1452c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1453c9de560dSAlex Tomas 1454c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1455c9de560dSAlex Tomas * allocated blocks for history */ 1456c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1457c9de560dSAlex Tomas 1458c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1459c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1460c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1461c9de560dSAlex Tomas 1462c3a326a6SAneesh Kumar K.V /* 1463c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 1464c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 1465c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 1466c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 1467c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 1468c3a326a6SAneesh Kumar K.V */ 1469c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1470c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1471c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1472c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 14738556e8f3SAneesh Kumar K.V /* on allocation we use ac to track the held semaphore */ 14748556e8f3SAneesh Kumar K.V ac->alloc_semp = e4b->alloc_semp; 14758556e8f3SAneesh Kumar K.V e4b->alloc_semp = NULL; 1476c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 14774ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 1478c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1479c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1480c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1481c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1482c9de560dSAlex Tomas } 1483c9de560dSAlex Tomas } 1484c9de560dSAlex Tomas 1485c9de560dSAlex Tomas /* 1486c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1487c9de560dSAlex Tomas */ 1488c9de560dSAlex Tomas 1489c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1490c9de560dSAlex Tomas struct ext4_buddy *e4b, 1491c9de560dSAlex Tomas int finish_group) 1492c9de560dSAlex Tomas { 1493c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1494c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1495c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1496c9de560dSAlex Tomas struct ext4_free_extent ex; 1497c9de560dSAlex Tomas int max; 1498c9de560dSAlex Tomas 1499032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 1500032115fcSAneesh Kumar K.V return; 1501c9de560dSAlex Tomas /* 1502c9de560dSAlex Tomas * We don't want to scan for a whole year 1503c9de560dSAlex Tomas */ 1504c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1505c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1506c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1507c9de560dSAlex Tomas return; 1508c9de560dSAlex Tomas } 1509c9de560dSAlex Tomas 1510c9de560dSAlex Tomas /* 1511c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1512c9de560dSAlex Tomas */ 1513c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1514c9de560dSAlex Tomas return; 1515c9de560dSAlex Tomas 1516c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1517c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1518c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1519c9de560dSAlex Tomas * when it was found (within this lock-unlock 1520c9de560dSAlex Tomas * period or not) */ 1521c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex); 1522c9de560dSAlex Tomas if (max >= gex->fe_len) { 1523c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1524c9de560dSAlex Tomas return; 1525c9de560dSAlex Tomas } 1526c9de560dSAlex Tomas } 1527c9de560dSAlex Tomas } 1528c9de560dSAlex Tomas 1529c9de560dSAlex Tomas /* 1530c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1531c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1532c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1533c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1534c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1535c9de560dSAlex Tomas * mballoc can't find good enough extent. 1536c9de560dSAlex Tomas * 1537c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1538c9de560dSAlex Tomas */ 1539c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1540c9de560dSAlex Tomas struct ext4_free_extent *ex, 1541c9de560dSAlex Tomas struct ext4_buddy *e4b) 1542c9de560dSAlex Tomas { 1543c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1544c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1545c9de560dSAlex Tomas 1546c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 15478d03c7a0SEric Sandeen BUG_ON(ex->fe_len > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 1548c9de560dSAlex Tomas BUG_ON(ex->fe_start >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 1549c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1550c9de560dSAlex Tomas 1551c9de560dSAlex Tomas ac->ac_found++; 1552c9de560dSAlex Tomas 1553c9de560dSAlex Tomas /* 1554c9de560dSAlex Tomas * The special case - take what you catch first 1555c9de560dSAlex Tomas */ 1556c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1557c9de560dSAlex Tomas *bex = *ex; 1558c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1559c9de560dSAlex Tomas return; 1560c9de560dSAlex Tomas } 1561c9de560dSAlex Tomas 1562c9de560dSAlex Tomas /* 1563c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1564c9de560dSAlex Tomas */ 1565c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1566c9de560dSAlex Tomas *bex = *ex; 1567c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1568c9de560dSAlex Tomas return; 1569c9de560dSAlex Tomas } 1570c9de560dSAlex Tomas 1571c9de560dSAlex Tomas /* 1572c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1573c9de560dSAlex Tomas */ 1574c9de560dSAlex Tomas if (bex->fe_len == 0) { 1575c9de560dSAlex Tomas *bex = *ex; 1576c9de560dSAlex Tomas return; 1577c9de560dSAlex Tomas } 1578c9de560dSAlex Tomas 1579c9de560dSAlex Tomas /* 1580c9de560dSAlex Tomas * If new found extent is better, store it in the context 1581c9de560dSAlex Tomas */ 1582c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1583c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1584c9de560dSAlex Tomas * larger than previous best one is better */ 1585c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1586c9de560dSAlex Tomas *bex = *ex; 1587c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1588c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1589c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1590c9de560dSAlex Tomas * smaller than previous one */ 1591c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1592c9de560dSAlex Tomas *bex = *ex; 1593c9de560dSAlex Tomas } 1594c9de560dSAlex Tomas 1595c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1596c9de560dSAlex Tomas } 1597c9de560dSAlex Tomas 1598089ceeccSEric Sandeen static noinline_for_stack 1599089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1600c9de560dSAlex Tomas struct ext4_buddy *e4b) 1601c9de560dSAlex Tomas { 1602c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1603c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1604c9de560dSAlex Tomas int max; 1605c9de560dSAlex Tomas int err; 1606c9de560dSAlex Tomas 1607c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1608c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1609c9de560dSAlex Tomas if (err) 1610c9de560dSAlex Tomas return err; 1611c9de560dSAlex Tomas 1612c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1613c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex); 1614c9de560dSAlex Tomas 1615c9de560dSAlex Tomas if (max > 0) { 1616c9de560dSAlex Tomas ac->ac_b_ex = ex; 1617c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1618c9de560dSAlex Tomas } 1619c9de560dSAlex Tomas 1620c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1621c9de560dSAlex Tomas ext4_mb_release_desc(e4b); 1622c9de560dSAlex Tomas 1623c9de560dSAlex Tomas return 0; 1624c9de560dSAlex Tomas } 1625c9de560dSAlex Tomas 1626089ceeccSEric Sandeen static noinline_for_stack 1627089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1628c9de560dSAlex Tomas struct ext4_buddy *e4b) 1629c9de560dSAlex Tomas { 1630c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1631c9de560dSAlex Tomas int max; 1632c9de560dSAlex Tomas int err; 1633c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1634c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 1635c9de560dSAlex Tomas struct ext4_free_extent ex; 1636c9de560dSAlex Tomas 1637c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1638c9de560dSAlex Tomas return 0; 1639c9de560dSAlex Tomas 1640c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1641c9de560dSAlex Tomas if (err) 1642c9de560dSAlex Tomas return err; 1643c9de560dSAlex Tomas 1644c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1645c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start, 1646c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1647c9de560dSAlex Tomas 1648c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1649c9de560dSAlex Tomas ext4_fsblk_t start; 1650c9de560dSAlex Tomas 1651c9de560dSAlex Tomas start = (e4b->bd_group * EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) + 1652c9de560dSAlex Tomas ex.fe_start + le32_to_cpu(es->s_first_data_block); 1653c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1654c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1655c9de560dSAlex Tomas ac->ac_found++; 1656c9de560dSAlex Tomas ac->ac_b_ex = ex; 1657c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1658c9de560dSAlex Tomas } 1659c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1660c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1661c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1662c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1663c9de560dSAlex Tomas ac->ac_found++; 1664c9de560dSAlex Tomas ac->ac_b_ex = ex; 1665c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1666c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1667c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1668c9de560dSAlex Tomas * number of blocks to an existing extent */ 1669c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1670c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1671c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1672c9de560dSAlex Tomas ac->ac_found++; 1673c9de560dSAlex Tomas ac->ac_b_ex = ex; 1674c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1675c9de560dSAlex Tomas } 1676c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1677c9de560dSAlex Tomas ext4_mb_release_desc(e4b); 1678c9de560dSAlex Tomas 1679c9de560dSAlex Tomas return 0; 1680c9de560dSAlex Tomas } 1681c9de560dSAlex Tomas 1682c9de560dSAlex Tomas /* 1683c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1684c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1685c9de560dSAlex Tomas */ 1686089ceeccSEric Sandeen static noinline_for_stack 1687089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1688c9de560dSAlex Tomas struct ext4_buddy *e4b) 1689c9de560dSAlex Tomas { 1690c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1691c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1692c9de560dSAlex Tomas void *buddy; 1693c9de560dSAlex Tomas int i; 1694c9de560dSAlex Tomas int k; 1695c9de560dSAlex Tomas int max; 1696c9de560dSAlex Tomas 1697c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1698c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1699c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1700c9de560dSAlex Tomas continue; 1701c9de560dSAlex Tomas 1702c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1703c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1704c9de560dSAlex Tomas 1705ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 1706c9de560dSAlex Tomas BUG_ON(k >= max); 1707c9de560dSAlex Tomas 1708c9de560dSAlex Tomas ac->ac_found++; 1709c9de560dSAlex Tomas 1710c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1711c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1712c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1713c9de560dSAlex Tomas 1714c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1715c9de560dSAlex Tomas 1716c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); 1717c9de560dSAlex Tomas 1718c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1719c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1720c9de560dSAlex Tomas 1721c9de560dSAlex Tomas break; 1722c9de560dSAlex Tomas } 1723c9de560dSAlex Tomas } 1724c9de560dSAlex Tomas 1725c9de560dSAlex Tomas /* 1726c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 1727c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 1728c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 1729c9de560dSAlex Tomas */ 1730089ceeccSEric Sandeen static noinline_for_stack 1731089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 1732c9de560dSAlex Tomas struct ext4_buddy *e4b) 1733c9de560dSAlex Tomas { 1734c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1735c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1736c9de560dSAlex Tomas struct ext4_free_extent ex; 1737c9de560dSAlex Tomas int i; 1738c9de560dSAlex Tomas int free; 1739c9de560dSAlex Tomas 1740c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 1741c9de560dSAlex Tomas BUG_ON(free <= 0); 1742c9de560dSAlex Tomas 1743c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 1744c9de560dSAlex Tomas 1745c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 1746ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 1747c9de560dSAlex Tomas EXT4_BLOCKS_PER_GROUP(sb), i); 1748c9de560dSAlex Tomas if (i >= EXT4_BLOCKS_PER_GROUP(sb)) { 174926346ff6SAneesh Kumar K.V /* 1750e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 175126346ff6SAneesh Kumar K.V * free blocks even though group info says we 175226346ff6SAneesh Kumar K.V * we have free blocks 175326346ff6SAneesh Kumar K.V */ 17545d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 17555d1b1b3fSAneesh Kumar K.V __func__, "%d free blocks as per " 1756fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 175726346ff6SAneesh Kumar K.V free); 1758c9de560dSAlex Tomas break; 1759c9de560dSAlex Tomas } 1760c9de560dSAlex Tomas 1761c9de560dSAlex Tomas mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex); 1762c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 176326346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 17645d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 17655d1b1b3fSAneesh Kumar K.V __func__, "%d free blocks as per " 1766fde4d95aSTheodore Ts'o "group info. But got %d blocks", 176726346ff6SAneesh Kumar K.V free, ex.fe_len); 1768e56eb659SAneesh Kumar K.V /* 1769e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 1770e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 1771e56eb659SAneesh Kumar K.V * without claiming the space. 1772e56eb659SAneesh Kumar K.V */ 1773e56eb659SAneesh Kumar K.V break; 177426346ff6SAneesh Kumar K.V } 1775c9de560dSAlex Tomas 1776c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 1777c9de560dSAlex Tomas 1778c9de560dSAlex Tomas i += ex.fe_len; 1779c9de560dSAlex Tomas free -= ex.fe_len; 1780c9de560dSAlex Tomas } 1781c9de560dSAlex Tomas 1782c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 1783c9de560dSAlex Tomas } 1784c9de560dSAlex Tomas 1785c9de560dSAlex Tomas /* 1786c9de560dSAlex Tomas * This is a special case for storages like raid5 1787c9de560dSAlex Tomas * we try to find stripe-aligned chunks for stripe-size requests 1788c9de560dSAlex Tomas * XXX should do so at least for multiples of stripe size as well 1789c9de560dSAlex Tomas */ 1790089ceeccSEric Sandeen static noinline_for_stack 1791089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 1792c9de560dSAlex Tomas struct ext4_buddy *e4b) 1793c9de560dSAlex Tomas { 1794c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1795c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 1796c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1797c9de560dSAlex Tomas struct ext4_free_extent ex; 1798c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 1799c9de560dSAlex Tomas ext4_fsblk_t a; 1800c9de560dSAlex Tomas ext4_grpblk_t i; 1801c9de560dSAlex Tomas int max; 1802c9de560dSAlex Tomas 1803c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 1804c9de560dSAlex Tomas 1805c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 1806c9de560dSAlex Tomas first_group_block = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb) 1807c9de560dSAlex Tomas + le32_to_cpu(sbi->s_es->s_first_data_block); 1808c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 1809c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 1810c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 1811c9de560dSAlex Tomas 1812c9de560dSAlex Tomas while (i < EXT4_BLOCKS_PER_GROUP(sb)) { 1813c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 1814c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, i, sbi->s_stripe, &ex); 1815c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 1816c9de560dSAlex Tomas ac->ac_found++; 1817c9de560dSAlex Tomas ac->ac_b_ex = ex; 1818c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1819c9de560dSAlex Tomas break; 1820c9de560dSAlex Tomas } 1821c9de560dSAlex Tomas } 1822c9de560dSAlex Tomas i += sbi->s_stripe; 1823c9de560dSAlex Tomas } 1824c9de560dSAlex Tomas } 1825c9de560dSAlex Tomas 1826c9de560dSAlex Tomas static int ext4_mb_good_group(struct ext4_allocation_context *ac, 1827c9de560dSAlex Tomas ext4_group_t group, int cr) 1828c9de560dSAlex Tomas { 1829c9de560dSAlex Tomas unsigned free, fragments; 1830c9de560dSAlex Tomas unsigned i, bits; 1831a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 1832c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1833c9de560dSAlex Tomas 1834c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 1835c9de560dSAlex Tomas BUG_ON(EXT4_MB_GRP_NEED_INIT(grp)); 1836c9de560dSAlex Tomas 1837c9de560dSAlex Tomas free = grp->bb_free; 1838c9de560dSAlex Tomas fragments = grp->bb_fragments; 1839c9de560dSAlex Tomas if (free == 0) 1840c9de560dSAlex Tomas return 0; 1841c9de560dSAlex Tomas if (fragments == 0) 1842c9de560dSAlex Tomas return 0; 1843c9de560dSAlex Tomas 1844c9de560dSAlex Tomas switch (cr) { 1845c9de560dSAlex Tomas case 0: 1846c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 1847c9de560dSAlex Tomas 1848a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 1849a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 1850a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 1851a4912123STheodore Ts'o ((group % flex_size) == 0)) 1852a4912123STheodore Ts'o return 0; 1853a4912123STheodore Ts'o 1854c9de560dSAlex Tomas bits = ac->ac_sb->s_blocksize_bits + 1; 1855c9de560dSAlex Tomas for (i = ac->ac_2order; i <= bits; i++) 1856c9de560dSAlex Tomas if (grp->bb_counters[i] > 0) 1857c9de560dSAlex Tomas return 1; 1858c9de560dSAlex Tomas break; 1859c9de560dSAlex Tomas case 1: 1860c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 1861c9de560dSAlex Tomas return 1; 1862c9de560dSAlex Tomas break; 1863c9de560dSAlex Tomas case 2: 1864c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 1865c9de560dSAlex Tomas return 1; 1866c9de560dSAlex Tomas break; 1867c9de560dSAlex Tomas case 3: 1868c9de560dSAlex Tomas return 1; 1869c9de560dSAlex Tomas default: 1870c9de560dSAlex Tomas BUG(); 1871c9de560dSAlex Tomas } 1872c9de560dSAlex Tomas 1873c9de560dSAlex Tomas return 0; 1874c9de560dSAlex Tomas } 1875c9de560dSAlex Tomas 1876920313a7SAneesh Kumar K.V /* 1877920313a7SAneesh Kumar K.V * lock the group_info alloc_sem of all the groups 1878920313a7SAneesh Kumar K.V * belonging to the same buddy cache page. This 1879920313a7SAneesh Kumar K.V * make sure other parallel operation on the buddy 1880920313a7SAneesh Kumar K.V * cache doesn't happen whild holding the buddy cache 1881920313a7SAneesh Kumar K.V * lock 1882920313a7SAneesh Kumar K.V */ 1883920313a7SAneesh Kumar K.V int ext4_mb_get_buddy_cache_lock(struct super_block *sb, ext4_group_t group) 1884920313a7SAneesh Kumar K.V { 1885920313a7SAneesh Kumar K.V int i; 1886920313a7SAneesh Kumar K.V int block, pnum; 1887920313a7SAneesh Kumar K.V int blocks_per_page; 1888920313a7SAneesh Kumar K.V int groups_per_page; 18898df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 1890920313a7SAneesh Kumar K.V ext4_group_t first_group; 1891920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1892920313a7SAneesh Kumar K.V 1893920313a7SAneesh Kumar K.V blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1894920313a7SAneesh Kumar K.V /* 1895920313a7SAneesh Kumar K.V * the buddy cache inode stores the block bitmap 1896920313a7SAneesh Kumar K.V * and buddy information in consecutive blocks. 1897920313a7SAneesh Kumar K.V * So for each group we need two blocks. 1898920313a7SAneesh Kumar K.V */ 1899920313a7SAneesh Kumar K.V block = group * 2; 1900920313a7SAneesh Kumar K.V pnum = block / blocks_per_page; 1901920313a7SAneesh Kumar K.V first_group = pnum * blocks_per_page / 2; 1902920313a7SAneesh Kumar K.V 1903920313a7SAneesh Kumar K.V groups_per_page = blocks_per_page >> 1; 1904920313a7SAneesh Kumar K.V if (groups_per_page == 0) 1905920313a7SAneesh Kumar K.V groups_per_page = 1; 1906920313a7SAneesh Kumar K.V /* read all groups the page covers into the cache */ 1907920313a7SAneesh Kumar K.V for (i = 0; i < groups_per_page; i++) { 1908920313a7SAneesh Kumar K.V 19098df9675fSTheodore Ts'o if ((first_group + i) >= ngroups) 1910920313a7SAneesh Kumar K.V break; 1911920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, first_group + i); 1912920313a7SAneesh Kumar K.V /* take all groups write allocation 1913920313a7SAneesh Kumar K.V * semaphore. This make sure there is 1914920313a7SAneesh Kumar K.V * no block allocation going on in any 1915920313a7SAneesh Kumar K.V * of that groups 1916920313a7SAneesh Kumar K.V */ 1917b7be019eSAneesh Kumar K.V down_write_nested(&grp->alloc_sem, i); 1918920313a7SAneesh Kumar K.V } 1919920313a7SAneesh Kumar K.V return i; 1920920313a7SAneesh Kumar K.V } 1921920313a7SAneesh Kumar K.V 1922920313a7SAneesh Kumar K.V void ext4_mb_put_buddy_cache_lock(struct super_block *sb, 1923920313a7SAneesh Kumar K.V ext4_group_t group, int locked_group) 1924920313a7SAneesh Kumar K.V { 1925920313a7SAneesh Kumar K.V int i; 1926920313a7SAneesh Kumar K.V int block, pnum; 1927920313a7SAneesh Kumar K.V int blocks_per_page; 1928920313a7SAneesh Kumar K.V ext4_group_t first_group; 1929920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1930920313a7SAneesh Kumar K.V 1931920313a7SAneesh Kumar K.V blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1932920313a7SAneesh Kumar K.V /* 1933920313a7SAneesh Kumar K.V * the buddy cache inode stores the block bitmap 1934920313a7SAneesh Kumar K.V * and buddy information in consecutive blocks. 1935920313a7SAneesh Kumar K.V * So for each group we need two blocks. 1936920313a7SAneesh Kumar K.V */ 1937920313a7SAneesh Kumar K.V block = group * 2; 1938920313a7SAneesh Kumar K.V pnum = block / blocks_per_page; 1939920313a7SAneesh Kumar K.V first_group = pnum * blocks_per_page / 2; 1940920313a7SAneesh Kumar K.V /* release locks on all the groups */ 1941920313a7SAneesh Kumar K.V for (i = 0; i < locked_group; i++) { 1942920313a7SAneesh Kumar K.V 1943920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, first_group + i); 1944920313a7SAneesh Kumar K.V /* take all groups write allocation 1945920313a7SAneesh Kumar K.V * semaphore. This make sure there is 1946920313a7SAneesh Kumar K.V * no block allocation going on in any 1947920313a7SAneesh Kumar K.V * of that groups 1948920313a7SAneesh Kumar K.V */ 1949920313a7SAneesh Kumar K.V up_write(&grp->alloc_sem); 1950920313a7SAneesh Kumar K.V } 1951920313a7SAneesh Kumar K.V 1952920313a7SAneesh Kumar K.V } 1953920313a7SAneesh Kumar K.V 19544ddfef7bSEric Sandeen static noinline_for_stack int 19554ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 1956c9de560dSAlex Tomas { 19578df9675fSTheodore Ts'o ext4_group_t ngroups, group, i; 1958c9de560dSAlex Tomas int cr; 1959c9de560dSAlex Tomas int err = 0; 1960c9de560dSAlex Tomas int bsbits; 1961c9de560dSAlex Tomas struct ext4_sb_info *sbi; 1962c9de560dSAlex Tomas struct super_block *sb; 1963c9de560dSAlex Tomas struct ext4_buddy e4b; 1964c9de560dSAlex Tomas 1965c9de560dSAlex Tomas sb = ac->ac_sb; 1966c9de560dSAlex Tomas sbi = EXT4_SB(sb); 19678df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 1968fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 1969fb0a387dSEric Sandeen if (!(EXT4_I(ac->ac_inode)->i_flags & EXT4_EXTENTS_FL)) 1970fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 1971fb0a387dSEric Sandeen 1972c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1973c9de560dSAlex Tomas 1974c9de560dSAlex Tomas /* first, try the goal */ 1975c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 1976c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 1977c9de560dSAlex Tomas goto out; 1978c9de560dSAlex Tomas 1979c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 1980c9de560dSAlex Tomas goto out; 1981c9de560dSAlex Tomas 1982c9de560dSAlex Tomas /* 1983c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 1984c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 1985c9de560dSAlex Tomas * try exact allocation using buddy. 1986c9de560dSAlex Tomas */ 1987c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 1988c9de560dSAlex Tomas ac->ac_2order = 0; 1989c9de560dSAlex Tomas /* 1990c9de560dSAlex Tomas * We search using buddy data only if the order of the request 1991c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 1992b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 1993c9de560dSAlex Tomas */ 1994c9de560dSAlex Tomas if (i >= sbi->s_mb_order2_reqs) { 1995c9de560dSAlex Tomas /* 1996c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 1997c9de560dSAlex Tomas */ 1998c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 1999c9de560dSAlex Tomas ac->ac_2order = i - 1; 2000c9de560dSAlex Tomas } 2001c9de560dSAlex Tomas 2002c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 2003c9de560dSAlex Tomas 20044ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 20054ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2006c9de560dSAlex Tomas /* TBD: may be hot point */ 2007c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2008c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2009c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2010c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2011c9de560dSAlex Tomas } 20124ba74d00STheodore Ts'o 2013c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2014c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2015c9de560dSAlex Tomas /* 2016c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2017c9de560dSAlex Tomas * cr == 3 try to get anything 2018c9de560dSAlex Tomas */ 2019c9de560dSAlex Tomas repeat: 2020c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2021c9de560dSAlex Tomas ac->ac_criteria = cr; 2022ed8f9c75SAneesh Kumar K.V /* 2023ed8f9c75SAneesh Kumar K.V * searching for the right group start 2024ed8f9c75SAneesh Kumar K.V * from the goal value specified 2025ed8f9c75SAneesh Kumar K.V */ 2026ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2027ed8f9c75SAneesh Kumar K.V 20288df9675fSTheodore Ts'o for (i = 0; i < ngroups; group++, i++) { 2029c9de560dSAlex Tomas struct ext4_group_info *grp; 2030c9de560dSAlex Tomas struct ext4_group_desc *desc; 2031c9de560dSAlex Tomas 20328df9675fSTheodore Ts'o if (group == ngroups) 2033c9de560dSAlex Tomas group = 0; 2034c9de560dSAlex Tomas 2035c9de560dSAlex Tomas /* quick check to skip empty groups */ 2036920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 2037c9de560dSAlex Tomas if (grp->bb_free == 0) 2038c9de560dSAlex Tomas continue; 2039c9de560dSAlex Tomas 2040c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2041c9de560dSAlex Tomas if (err) 2042c9de560dSAlex Tomas goto out; 2043c9de560dSAlex Tomas 2044c9de560dSAlex Tomas ext4_lock_group(sb, group); 2045c9de560dSAlex Tomas if (!ext4_mb_good_group(ac, group, cr)) { 2046c9de560dSAlex Tomas /* someone did allocation from this group */ 2047c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2048c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2049c9de560dSAlex Tomas continue; 2050c9de560dSAlex Tomas } 2051c9de560dSAlex Tomas 2052c9de560dSAlex Tomas ac->ac_groups_scanned++; 2053c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, group, NULL); 205475507efbSTheodore Ts'o if (cr == 0) 2055c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2056c9de560dSAlex Tomas else if (cr == 1 && 2057c9de560dSAlex Tomas ac->ac_g_ex.fe_len == sbi->s_stripe) 2058c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2059c9de560dSAlex Tomas else 2060c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2061c9de560dSAlex Tomas 2062c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2063c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2064c9de560dSAlex Tomas 2065c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2066c9de560dSAlex Tomas break; 2067c9de560dSAlex Tomas } 2068c9de560dSAlex Tomas } 2069c9de560dSAlex Tomas 2070c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2071c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2072c9de560dSAlex Tomas /* 2073c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2074c9de560dSAlex Tomas * the best chunk we've found so far 2075c9de560dSAlex Tomas */ 2076c9de560dSAlex Tomas 2077c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2078c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2079c9de560dSAlex Tomas /* 2080c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2081c9de560dSAlex Tomas * The only thing we can do is just take first 2082c9de560dSAlex Tomas * found block(s) 2083c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2084c9de560dSAlex Tomas */ 2085c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2086c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2087c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2088c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2089c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2090c9de560dSAlex Tomas cr = 3; 2091c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2092c9de560dSAlex Tomas goto repeat; 2093c9de560dSAlex Tomas } 2094c9de560dSAlex Tomas } 2095c9de560dSAlex Tomas out: 2096c9de560dSAlex Tomas return err; 2097c9de560dSAlex Tomas } 2098c9de560dSAlex Tomas 2099c9de560dSAlex Tomas #ifdef EXT4_MB_HISTORY 2100c9de560dSAlex Tomas struct ext4_mb_proc_session { 2101c9de560dSAlex Tomas struct ext4_mb_history *history; 2102c9de560dSAlex Tomas struct super_block *sb; 2103c9de560dSAlex Tomas int start; 2104c9de560dSAlex Tomas int max; 2105c9de560dSAlex Tomas }; 2106c9de560dSAlex Tomas 2107c9de560dSAlex Tomas static void *ext4_mb_history_skip_empty(struct ext4_mb_proc_session *s, 2108c9de560dSAlex Tomas struct ext4_mb_history *hs, 2109c9de560dSAlex Tomas int first) 2110c9de560dSAlex Tomas { 2111c9de560dSAlex Tomas if (hs == s->history + s->max) 2112c9de560dSAlex Tomas hs = s->history; 2113c9de560dSAlex Tomas if (!first && hs == s->history + s->start) 2114c9de560dSAlex Tomas return NULL; 2115c9de560dSAlex Tomas while (hs->orig.fe_len == 0) { 2116c9de560dSAlex Tomas hs++; 2117c9de560dSAlex Tomas if (hs == s->history + s->max) 2118c9de560dSAlex Tomas hs = s->history; 2119c9de560dSAlex Tomas if (hs == s->history + s->start) 2120c9de560dSAlex Tomas return NULL; 2121c9de560dSAlex Tomas } 2122c9de560dSAlex Tomas return hs; 2123c9de560dSAlex Tomas } 2124c9de560dSAlex Tomas 2125c9de560dSAlex Tomas static void *ext4_mb_seq_history_start(struct seq_file *seq, loff_t *pos) 2126c9de560dSAlex Tomas { 2127c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2128c9de560dSAlex Tomas struct ext4_mb_history *hs; 2129c9de560dSAlex Tomas int l = *pos; 2130c9de560dSAlex Tomas 2131c9de560dSAlex Tomas if (l == 0) 2132c9de560dSAlex Tomas return SEQ_START_TOKEN; 2133c9de560dSAlex Tomas hs = ext4_mb_history_skip_empty(s, s->history + s->start, 1); 2134c9de560dSAlex Tomas if (!hs) 2135c9de560dSAlex Tomas return NULL; 2136c9de560dSAlex Tomas while (--l && (hs = ext4_mb_history_skip_empty(s, ++hs, 0)) != NULL); 2137c9de560dSAlex Tomas return hs; 2138c9de560dSAlex Tomas } 2139c9de560dSAlex Tomas 2140c9de560dSAlex Tomas static void *ext4_mb_seq_history_next(struct seq_file *seq, void *v, 2141c9de560dSAlex Tomas loff_t *pos) 2142c9de560dSAlex Tomas { 2143c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2144c9de560dSAlex Tomas struct ext4_mb_history *hs = v; 2145c9de560dSAlex Tomas 2146c9de560dSAlex Tomas ++*pos; 2147c9de560dSAlex Tomas if (v == SEQ_START_TOKEN) 2148c9de560dSAlex Tomas return ext4_mb_history_skip_empty(s, s->history + s->start, 1); 2149c9de560dSAlex Tomas else 2150c9de560dSAlex Tomas return ext4_mb_history_skip_empty(s, ++hs, 0); 2151c9de560dSAlex Tomas } 2152c9de560dSAlex Tomas 2153c9de560dSAlex Tomas static int ext4_mb_seq_history_show(struct seq_file *seq, void *v) 2154c9de560dSAlex Tomas { 2155c9de560dSAlex Tomas char buf[25], buf2[25], buf3[25], *fmt; 2156c9de560dSAlex Tomas struct ext4_mb_history *hs = v; 2157c9de560dSAlex Tomas 2158c9de560dSAlex Tomas if (v == SEQ_START_TOKEN) { 2159c9de560dSAlex Tomas seq_printf(seq, "%-5s %-8s %-23s %-23s %-23s %-5s " 21600ef90db9STheodore Ts'o "%-5s %-2s %-6s %-5s %-5s %-6s\n", 2161c9de560dSAlex Tomas "pid", "inode", "original", "goal", "result", "found", 2162c9de560dSAlex Tomas "grps", "cr", "flags", "merge", "tail", "broken"); 2163c9de560dSAlex Tomas return 0; 2164c9de560dSAlex Tomas } 2165c9de560dSAlex Tomas 2166c9de560dSAlex Tomas if (hs->op == EXT4_MB_HISTORY_ALLOC) { 2167c9de560dSAlex Tomas fmt = "%-5u %-8u %-23s %-23s %-23s %-5u %-5u %-2u " 21680ef90db9STheodore Ts'o "0x%04x %-5s %-5u %-6u\n"; 2169a9df9a49STheodore Ts'o sprintf(buf2, "%u/%d/%u@%u", hs->result.fe_group, 2170c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len, 2171c9de560dSAlex Tomas hs->result.fe_logical); 2172a9df9a49STheodore Ts'o sprintf(buf, "%u/%d/%u@%u", hs->orig.fe_group, 2173c9de560dSAlex Tomas hs->orig.fe_start, hs->orig.fe_len, 2174c9de560dSAlex Tomas hs->orig.fe_logical); 2175a9df9a49STheodore Ts'o sprintf(buf3, "%u/%d/%u@%u", hs->goal.fe_group, 2176c9de560dSAlex Tomas hs->goal.fe_start, hs->goal.fe_len, 2177c9de560dSAlex Tomas hs->goal.fe_logical); 2178c9de560dSAlex Tomas seq_printf(seq, fmt, hs->pid, hs->ino, buf, buf3, buf2, 2179c9de560dSAlex Tomas hs->found, hs->groups, hs->cr, hs->flags, 2180c9de560dSAlex Tomas hs->merged ? "M" : "", hs->tail, 2181c9de560dSAlex Tomas hs->buddy ? 1 << hs->buddy : 0); 2182c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_PREALLOC) { 2183c9de560dSAlex Tomas fmt = "%-5u %-8u %-23s %-23s %-23s\n"; 2184a9df9a49STheodore Ts'o sprintf(buf2, "%u/%d/%u@%u", hs->result.fe_group, 2185c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len, 2186c9de560dSAlex Tomas hs->result.fe_logical); 2187a9df9a49STheodore Ts'o sprintf(buf, "%u/%d/%u@%u", hs->orig.fe_group, 2188c9de560dSAlex Tomas hs->orig.fe_start, hs->orig.fe_len, 2189c9de560dSAlex Tomas hs->orig.fe_logical); 2190c9de560dSAlex Tomas seq_printf(seq, fmt, hs->pid, hs->ino, buf, "", buf2); 2191c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_DISCARD) { 2192a9df9a49STheodore Ts'o sprintf(buf2, "%u/%d/%u", hs->result.fe_group, 2193c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len); 2194c9de560dSAlex Tomas seq_printf(seq, "%-5u %-8u %-23s discard\n", 2195c9de560dSAlex Tomas hs->pid, hs->ino, buf2); 2196c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_FREE) { 2197a9df9a49STheodore Ts'o sprintf(buf2, "%u/%d/%u", hs->result.fe_group, 2198c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len); 2199c9de560dSAlex Tomas seq_printf(seq, "%-5u %-8u %-23s free\n", 2200c9de560dSAlex Tomas hs->pid, hs->ino, buf2); 2201c9de560dSAlex Tomas } 2202c9de560dSAlex Tomas return 0; 2203c9de560dSAlex Tomas } 2204c9de560dSAlex Tomas 2205c9de560dSAlex Tomas static void ext4_mb_seq_history_stop(struct seq_file *seq, void *v) 2206c9de560dSAlex Tomas { 2207c9de560dSAlex Tomas } 2208c9de560dSAlex Tomas 22097f1346a9STobias Klauser static const struct seq_operations ext4_mb_seq_history_ops = { 2210c9de560dSAlex Tomas .start = ext4_mb_seq_history_start, 2211c9de560dSAlex Tomas .next = ext4_mb_seq_history_next, 2212c9de560dSAlex Tomas .stop = ext4_mb_seq_history_stop, 2213c9de560dSAlex Tomas .show = ext4_mb_seq_history_show, 2214c9de560dSAlex Tomas }; 2215c9de560dSAlex Tomas 2216c9de560dSAlex Tomas static int ext4_mb_seq_history_open(struct inode *inode, struct file *file) 2217c9de560dSAlex Tomas { 2218c9de560dSAlex Tomas struct super_block *sb = PDE(inode)->data; 2219c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2220c9de560dSAlex Tomas struct ext4_mb_proc_session *s; 2221c9de560dSAlex Tomas int rc; 2222c9de560dSAlex Tomas int size; 2223c9de560dSAlex Tomas 222474767c5aSShen Feng if (unlikely(sbi->s_mb_history == NULL)) 222574767c5aSShen Feng return -ENOMEM; 2226c9de560dSAlex Tomas s = kmalloc(sizeof(*s), GFP_KERNEL); 2227c9de560dSAlex Tomas if (s == NULL) 2228c9de560dSAlex Tomas return -ENOMEM; 2229c9de560dSAlex Tomas s->sb = sb; 2230c9de560dSAlex Tomas size = sizeof(struct ext4_mb_history) * sbi->s_mb_history_max; 2231c9de560dSAlex Tomas s->history = kmalloc(size, GFP_KERNEL); 2232c9de560dSAlex Tomas if (s->history == NULL) { 2233c9de560dSAlex Tomas kfree(s); 2234c9de560dSAlex Tomas return -ENOMEM; 2235c9de560dSAlex Tomas } 2236c9de560dSAlex Tomas 2237c9de560dSAlex Tomas spin_lock(&sbi->s_mb_history_lock); 2238c9de560dSAlex Tomas memcpy(s->history, sbi->s_mb_history, size); 2239c9de560dSAlex Tomas s->max = sbi->s_mb_history_max; 2240c9de560dSAlex Tomas s->start = sbi->s_mb_history_cur % s->max; 2241c9de560dSAlex Tomas spin_unlock(&sbi->s_mb_history_lock); 2242c9de560dSAlex Tomas 2243c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_history_ops); 2244c9de560dSAlex Tomas if (rc == 0) { 2245c9de560dSAlex Tomas struct seq_file *m = (struct seq_file *)file->private_data; 2246c9de560dSAlex Tomas m->private = s; 2247c9de560dSAlex Tomas } else { 2248c9de560dSAlex Tomas kfree(s->history); 2249c9de560dSAlex Tomas kfree(s); 2250c9de560dSAlex Tomas } 2251c9de560dSAlex Tomas return rc; 2252c9de560dSAlex Tomas 2253c9de560dSAlex Tomas } 2254c9de560dSAlex Tomas 2255c9de560dSAlex Tomas static int ext4_mb_seq_history_release(struct inode *inode, struct file *file) 2256c9de560dSAlex Tomas { 2257c9de560dSAlex Tomas struct seq_file *seq = (struct seq_file *)file->private_data; 2258c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2259c9de560dSAlex Tomas kfree(s->history); 2260c9de560dSAlex Tomas kfree(s); 2261c9de560dSAlex Tomas return seq_release(inode, file); 2262c9de560dSAlex Tomas } 2263c9de560dSAlex Tomas 2264c9de560dSAlex Tomas static ssize_t ext4_mb_seq_history_write(struct file *file, 2265c9de560dSAlex Tomas const char __user *buffer, 2266c9de560dSAlex Tomas size_t count, loff_t *ppos) 2267c9de560dSAlex Tomas { 2268c9de560dSAlex Tomas struct seq_file *seq = (struct seq_file *)file->private_data; 2269c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2270c9de560dSAlex Tomas struct super_block *sb = s->sb; 2271c9de560dSAlex Tomas char str[32]; 2272c9de560dSAlex Tomas int value; 2273c9de560dSAlex Tomas 2274c9de560dSAlex Tomas if (count >= sizeof(str)) { 2275c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: %s string too long, max %u bytes\n", 2276c9de560dSAlex Tomas "mb_history", (int)sizeof(str)); 2277c9de560dSAlex Tomas return -EOVERFLOW; 2278c9de560dSAlex Tomas } 2279c9de560dSAlex Tomas 2280c9de560dSAlex Tomas if (copy_from_user(str, buffer, count)) 2281c9de560dSAlex Tomas return -EFAULT; 2282c9de560dSAlex Tomas 2283c9de560dSAlex Tomas value = simple_strtol(str, NULL, 0); 2284c9de560dSAlex Tomas if (value < 0) 2285c9de560dSAlex Tomas return -ERANGE; 2286c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_history_filter = value; 2287c9de560dSAlex Tomas 2288c9de560dSAlex Tomas return count; 2289c9de560dSAlex Tomas } 2290c9de560dSAlex Tomas 22917f1346a9STobias Klauser static const struct file_operations ext4_mb_seq_history_fops = { 2292c9de560dSAlex Tomas .owner = THIS_MODULE, 2293c9de560dSAlex Tomas .open = ext4_mb_seq_history_open, 2294c9de560dSAlex Tomas .read = seq_read, 2295c9de560dSAlex Tomas .write = ext4_mb_seq_history_write, 2296c9de560dSAlex Tomas .llseek = seq_lseek, 2297c9de560dSAlex Tomas .release = ext4_mb_seq_history_release, 2298c9de560dSAlex Tomas }; 2299c9de560dSAlex Tomas 2300c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2301c9de560dSAlex Tomas { 2302c9de560dSAlex Tomas struct super_block *sb = seq->private; 2303c9de560dSAlex Tomas ext4_group_t group; 2304c9de560dSAlex Tomas 23058df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2306c9de560dSAlex Tomas return NULL; 2307c9de560dSAlex Tomas group = *pos + 1; 2308a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2309c9de560dSAlex Tomas } 2310c9de560dSAlex Tomas 2311c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2312c9de560dSAlex Tomas { 2313c9de560dSAlex Tomas struct super_block *sb = seq->private; 2314c9de560dSAlex Tomas ext4_group_t group; 2315c9de560dSAlex Tomas 2316c9de560dSAlex Tomas ++*pos; 23178df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2318c9de560dSAlex Tomas return NULL; 2319c9de560dSAlex Tomas group = *pos + 1; 2320a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2321c9de560dSAlex Tomas } 2322c9de560dSAlex Tomas 2323c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2324c9de560dSAlex Tomas { 2325c9de560dSAlex Tomas struct super_block *sb = seq->private; 2326a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2327c9de560dSAlex Tomas int i; 2328c9de560dSAlex Tomas int err; 2329c9de560dSAlex Tomas struct ext4_buddy e4b; 2330c9de560dSAlex Tomas struct sg { 2331c9de560dSAlex Tomas struct ext4_group_info info; 2332a36b4498SEric Sandeen ext4_grpblk_t counters[16]; 2333c9de560dSAlex Tomas } sg; 2334c9de560dSAlex Tomas 2335c9de560dSAlex Tomas group--; 2336c9de560dSAlex Tomas if (group == 0) 2337c9de560dSAlex Tomas seq_printf(seq, "#%-5s: %-5s %-5s %-5s " 2338c9de560dSAlex Tomas "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s " 2339c9de560dSAlex Tomas "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n", 2340c9de560dSAlex Tomas "group", "free", "frags", "first", 2341c9de560dSAlex Tomas "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6", 2342c9de560dSAlex Tomas "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13"); 2343c9de560dSAlex Tomas 2344c9de560dSAlex Tomas i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2345c9de560dSAlex Tomas sizeof(struct ext4_group_info); 2346c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2347c9de560dSAlex Tomas if (err) { 2348a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2349c9de560dSAlex Tomas return 0; 2350c9de560dSAlex Tomas } 2351c9de560dSAlex Tomas ext4_lock_group(sb, group); 2352c9de560dSAlex Tomas memcpy(&sg, ext4_get_group_info(sb, group), i); 2353c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2354c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2355c9de560dSAlex Tomas 2356a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2357c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2358c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 2359c9de560dSAlex Tomas seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ? 2360c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2361c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2362c9de560dSAlex Tomas 2363c9de560dSAlex Tomas return 0; 2364c9de560dSAlex Tomas } 2365c9de560dSAlex Tomas 2366c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2367c9de560dSAlex Tomas { 2368c9de560dSAlex Tomas } 2369c9de560dSAlex Tomas 23707f1346a9STobias Klauser static const struct seq_operations ext4_mb_seq_groups_ops = { 2371c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2372c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2373c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2374c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2375c9de560dSAlex Tomas }; 2376c9de560dSAlex Tomas 2377c9de560dSAlex Tomas static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file) 2378c9de560dSAlex Tomas { 2379c9de560dSAlex Tomas struct super_block *sb = PDE(inode)->data; 2380c9de560dSAlex Tomas int rc; 2381c9de560dSAlex Tomas 2382c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_groups_ops); 2383c9de560dSAlex Tomas if (rc == 0) { 2384c9de560dSAlex Tomas struct seq_file *m = (struct seq_file *)file->private_data; 2385c9de560dSAlex Tomas m->private = sb; 2386c9de560dSAlex Tomas } 2387c9de560dSAlex Tomas return rc; 2388c9de560dSAlex Tomas 2389c9de560dSAlex Tomas } 2390c9de560dSAlex Tomas 23917f1346a9STobias Klauser static const struct file_operations ext4_mb_seq_groups_fops = { 2392c9de560dSAlex Tomas .owner = THIS_MODULE, 2393c9de560dSAlex Tomas .open = ext4_mb_seq_groups_open, 2394c9de560dSAlex Tomas .read = seq_read, 2395c9de560dSAlex Tomas .llseek = seq_lseek, 2396c9de560dSAlex Tomas .release = seq_release, 2397c9de560dSAlex Tomas }; 2398c9de560dSAlex Tomas 2399c9de560dSAlex Tomas static void ext4_mb_history_release(struct super_block *sb) 2400c9de560dSAlex Tomas { 2401c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2402c9de560dSAlex Tomas 24039f6200bbSTheodore Ts'o if (sbi->s_proc != NULL) { 24049f6200bbSTheodore Ts'o remove_proc_entry("mb_groups", sbi->s_proc); 2405f4033903SCurt Wohlgemuth if (sbi->s_mb_history_max) 24069f6200bbSTheodore Ts'o remove_proc_entry("mb_history", sbi->s_proc); 24079f6200bbSTheodore Ts'o } 2408c9de560dSAlex Tomas kfree(sbi->s_mb_history); 2409c9de560dSAlex Tomas } 2410c9de560dSAlex Tomas 2411c9de560dSAlex Tomas static void ext4_mb_history_init(struct super_block *sb) 2412c9de560dSAlex Tomas { 2413c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2414c9de560dSAlex Tomas int i; 2415c9de560dSAlex Tomas 24169f6200bbSTheodore Ts'o if (sbi->s_proc != NULL) { 2417f4033903SCurt Wohlgemuth if (sbi->s_mb_history_max) 24189f6200bbSTheodore Ts'o proc_create_data("mb_history", S_IRUGO, sbi->s_proc, 241946fe74f2SDenis V. Lunev &ext4_mb_seq_history_fops, sb); 24209f6200bbSTheodore Ts'o proc_create_data("mb_groups", S_IRUGO, sbi->s_proc, 242146fe74f2SDenis V. Lunev &ext4_mb_seq_groups_fops, sb); 2422c9de560dSAlex Tomas } 2423c9de560dSAlex Tomas 2424c9de560dSAlex Tomas sbi->s_mb_history_cur = 0; 2425c9de560dSAlex Tomas spin_lock_init(&sbi->s_mb_history_lock); 2426c9de560dSAlex Tomas i = sbi->s_mb_history_max * sizeof(struct ext4_mb_history); 2427f4033903SCurt Wohlgemuth sbi->s_mb_history = i ? kzalloc(i, GFP_KERNEL) : NULL; 2428c9de560dSAlex Tomas /* if we can't allocate history, then we simple won't use it */ 2429c9de560dSAlex Tomas } 2430c9de560dSAlex Tomas 24314ddfef7bSEric Sandeen static noinline_for_stack void 24324ddfef7bSEric Sandeen ext4_mb_store_history(struct ext4_allocation_context *ac) 2433c9de560dSAlex Tomas { 2434c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2435c9de560dSAlex Tomas struct ext4_mb_history h; 2436c9de560dSAlex Tomas 2437f4033903SCurt Wohlgemuth if (sbi->s_mb_history == NULL) 2438c9de560dSAlex Tomas return; 2439c9de560dSAlex Tomas 2440c9de560dSAlex Tomas if (!(ac->ac_op & sbi->s_mb_history_filter)) 2441c9de560dSAlex Tomas return; 2442c9de560dSAlex Tomas 2443c9de560dSAlex Tomas h.op = ac->ac_op; 2444c9de560dSAlex Tomas h.pid = current->pid; 2445c9de560dSAlex Tomas h.ino = ac->ac_inode ? ac->ac_inode->i_ino : 0; 2446c9de560dSAlex Tomas h.orig = ac->ac_o_ex; 2447c9de560dSAlex Tomas h.result = ac->ac_b_ex; 2448c9de560dSAlex Tomas h.flags = ac->ac_flags; 2449c9de560dSAlex Tomas h.found = ac->ac_found; 2450c9de560dSAlex Tomas h.groups = ac->ac_groups_scanned; 2451c9de560dSAlex Tomas h.cr = ac->ac_criteria; 2452c9de560dSAlex Tomas h.tail = ac->ac_tail; 2453c9de560dSAlex Tomas h.buddy = ac->ac_buddy; 2454c9de560dSAlex Tomas h.merged = 0; 2455c9de560dSAlex Tomas if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) { 2456c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 2457c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 2458c9de560dSAlex Tomas h.merged = 1; 2459c9de560dSAlex Tomas h.goal = ac->ac_g_ex; 2460c9de560dSAlex Tomas h.result = ac->ac_f_ex; 2461c9de560dSAlex Tomas } 2462c9de560dSAlex Tomas 2463c9de560dSAlex Tomas spin_lock(&sbi->s_mb_history_lock); 2464c9de560dSAlex Tomas memcpy(sbi->s_mb_history + sbi->s_mb_history_cur, &h, sizeof(h)); 2465c9de560dSAlex Tomas if (++sbi->s_mb_history_cur >= sbi->s_mb_history_max) 2466c9de560dSAlex Tomas sbi->s_mb_history_cur = 0; 2467c9de560dSAlex Tomas spin_unlock(&sbi->s_mb_history_lock); 2468c9de560dSAlex Tomas } 2469c9de560dSAlex Tomas 2470c9de560dSAlex Tomas #else 2471c9de560dSAlex Tomas #define ext4_mb_history_release(sb) 2472c9de560dSAlex Tomas #define ext4_mb_history_init(sb) 2473c9de560dSAlex Tomas #endif 2474c9de560dSAlex Tomas 24755f21b0e6SFrederic Bohe 24765f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 2477920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 24785f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 24795f21b0e6SFrederic Bohe { 24805f21b0e6SFrederic Bohe int i, len; 24815f21b0e6SFrederic Bohe int metalen = 0; 24825f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 24835f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 24845f21b0e6SFrederic Bohe 24855f21b0e6SFrederic Bohe /* 24865f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 24875f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 24885f21b0e6SFrederic Bohe * to ext4_group_info structures 24895f21b0e6SFrederic Bohe */ 24905f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 24915f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 24925f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 24935f21b0e6SFrederic Bohe meta_group_info = kmalloc(metalen, GFP_KERNEL); 24945f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 24955f21b0e6SFrederic Bohe printk(KERN_ERR "EXT4-fs: can't allocate mem for a " 24965f21b0e6SFrederic Bohe "buddy group\n"); 24975f21b0e6SFrederic Bohe goto exit_meta_group_info; 24985f21b0e6SFrederic Bohe } 24995f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = 25005f21b0e6SFrederic Bohe meta_group_info; 25015f21b0e6SFrederic Bohe } 25025f21b0e6SFrederic Bohe 25035f21b0e6SFrederic Bohe /* 25045f21b0e6SFrederic Bohe * calculate needed size. if change bb_counters size, 25055f21b0e6SFrederic Bohe * don't forget about ext4_mb_generate_buddy() 25065f21b0e6SFrederic Bohe */ 25075f21b0e6SFrederic Bohe len = offsetof(typeof(**meta_group_info), 25085f21b0e6SFrederic Bohe bb_counters[sb->s_blocksize_bits + 2]); 25095f21b0e6SFrederic Bohe 25105f21b0e6SFrederic Bohe meta_group_info = 25115f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]; 25125f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 25135f21b0e6SFrederic Bohe 25145f21b0e6SFrederic Bohe meta_group_info[i] = kzalloc(len, GFP_KERNEL); 25155f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 25165f21b0e6SFrederic Bohe printk(KERN_ERR "EXT4-fs: can't allocate buddy mem\n"); 25175f21b0e6SFrederic Bohe goto exit_group_info; 25185f21b0e6SFrederic Bohe } 25195f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 25205f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 25215f21b0e6SFrederic Bohe 25225f21b0e6SFrederic Bohe /* 25235f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 25245f21b0e6SFrederic Bohe * empty groups without initialization 25255f21b0e6SFrederic Bohe */ 25265f21b0e6SFrederic Bohe if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 25275f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 25285f21b0e6SFrederic Bohe ext4_free_blocks_after_init(sb, group, desc); 25295f21b0e6SFrederic Bohe } else { 25305f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2531560671a0SAneesh Kumar K.V ext4_free_blks_count(sb, desc); 25325f21b0e6SFrederic Bohe } 25335f21b0e6SFrederic Bohe 25345f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 2535920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 25365a4a7989SJoe Perches meta_group_info[i]->bb_free_root.rb_node = NULL; 25375f21b0e6SFrederic Bohe 25385f21b0e6SFrederic Bohe #ifdef DOUBLE_CHECK 25395f21b0e6SFrederic Bohe { 25405f21b0e6SFrederic Bohe struct buffer_head *bh; 25415f21b0e6SFrederic Bohe meta_group_info[i]->bb_bitmap = 25425f21b0e6SFrederic Bohe kmalloc(sb->s_blocksize, GFP_KERNEL); 25435f21b0e6SFrederic Bohe BUG_ON(meta_group_info[i]->bb_bitmap == NULL); 25445f21b0e6SFrederic Bohe bh = ext4_read_block_bitmap(sb, group); 25455f21b0e6SFrederic Bohe BUG_ON(bh == NULL); 25465f21b0e6SFrederic Bohe memcpy(meta_group_info[i]->bb_bitmap, bh->b_data, 25475f21b0e6SFrederic Bohe sb->s_blocksize); 25485f21b0e6SFrederic Bohe put_bh(bh); 25495f21b0e6SFrederic Bohe } 25505f21b0e6SFrederic Bohe #endif 25515f21b0e6SFrederic Bohe 25525f21b0e6SFrederic Bohe return 0; 25535f21b0e6SFrederic Bohe 25545f21b0e6SFrederic Bohe exit_group_info: 25555f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 25565f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) 25575f21b0e6SFrederic Bohe kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]); 25585f21b0e6SFrederic Bohe exit_meta_group_info: 25595f21b0e6SFrederic Bohe return -ENOMEM; 25605f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 25615f21b0e6SFrederic Bohe 2562c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2563c9de560dSAlex Tomas { 25648df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2565c9de560dSAlex Tomas ext4_group_t i; 2566c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 25675f21b0e6SFrederic Bohe struct ext4_super_block *es = sbi->s_es; 25685f21b0e6SFrederic Bohe int num_meta_group_infos; 25695f21b0e6SFrederic Bohe int num_meta_group_infos_max; 25705f21b0e6SFrederic Bohe int array_size; 25715f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 2572c9de560dSAlex Tomas 25735f21b0e6SFrederic Bohe /* This is the number of blocks used by GDT */ 25748df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 25755f21b0e6SFrederic Bohe 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); 25765f21b0e6SFrederic Bohe 25775f21b0e6SFrederic Bohe /* 25785f21b0e6SFrederic Bohe * This is the total number of blocks used by GDT including 25795f21b0e6SFrederic Bohe * the number of reserved blocks for GDT. 25805f21b0e6SFrederic Bohe * The s_group_info array is allocated with this value 25815f21b0e6SFrederic Bohe * to allow a clean online resize without a complex 25825f21b0e6SFrederic Bohe * manipulation of pointer. 25835f21b0e6SFrederic Bohe * The drawback is the unused memory when no resize 25845f21b0e6SFrederic Bohe * occurs but it's very low in terms of pages 25855f21b0e6SFrederic Bohe * (see comments below) 25865f21b0e6SFrederic Bohe * Need to handle this properly when META_BG resizing is allowed 25875f21b0e6SFrederic Bohe */ 25885f21b0e6SFrederic Bohe num_meta_group_infos_max = num_meta_group_infos + 25895f21b0e6SFrederic Bohe le16_to_cpu(es->s_reserved_gdt_blocks); 25905f21b0e6SFrederic Bohe 25915f21b0e6SFrederic Bohe /* 25925f21b0e6SFrederic Bohe * array_size is the size of s_group_info array. We round it 25935f21b0e6SFrederic Bohe * to the next power of two because this approximation is done 25945f21b0e6SFrederic Bohe * internally by kmalloc so we can have some more memory 25955f21b0e6SFrederic Bohe * for free here (e.g. may be used for META_BG resize). 25965f21b0e6SFrederic Bohe */ 25975f21b0e6SFrederic Bohe array_size = 1; 25985f21b0e6SFrederic Bohe while (array_size < sizeof(*sbi->s_group_info) * 25995f21b0e6SFrederic Bohe num_meta_group_infos_max) 26005f21b0e6SFrederic Bohe array_size = array_size << 1; 2601c9de560dSAlex Tomas /* An 8TB filesystem with 64-bit pointers requires a 4096 byte 2602c9de560dSAlex Tomas * kmalloc. A 128kb malloc should suffice for a 256TB filesystem. 2603c9de560dSAlex Tomas * So a two level scheme suffices for now. */ 26045f21b0e6SFrederic Bohe sbi->s_group_info = kmalloc(array_size, GFP_KERNEL); 2605c9de560dSAlex Tomas if (sbi->s_group_info == NULL) { 2606c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't allocate buddy meta group\n"); 2607c9de560dSAlex Tomas return -ENOMEM; 2608c9de560dSAlex Tomas } 2609c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2610c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 2611c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't get new inode\n"); 2612c9de560dSAlex Tomas goto err_freesgi; 2613c9de560dSAlex Tomas } 2614c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 26158df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2616c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2617c9de560dSAlex Tomas if (desc == NULL) { 2618c9de560dSAlex Tomas printk(KERN_ERR 2619a9df9a49STheodore Ts'o "EXT4-fs: can't read descriptor %u\n", i); 2620c9de560dSAlex Tomas goto err_freebuddy; 2621c9de560dSAlex Tomas } 26225f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 26235f21b0e6SFrederic Bohe goto err_freebuddy; 2624c9de560dSAlex Tomas } 2625c9de560dSAlex Tomas 2626c9de560dSAlex Tomas return 0; 2627c9de560dSAlex Tomas 2628c9de560dSAlex Tomas err_freebuddy: 2629f1fa3342SRoel Kluin while (i-- > 0) 2630c9de560dSAlex Tomas kfree(ext4_get_group_info(sb, i)); 2631c9de560dSAlex Tomas i = num_meta_group_infos; 2632f1fa3342SRoel Kluin while (i-- > 0) 2633c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2634c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2635c9de560dSAlex Tomas err_freesgi: 2636c9de560dSAlex Tomas kfree(sbi->s_group_info); 2637c9de560dSAlex Tomas return -ENOMEM; 2638c9de560dSAlex Tomas } 2639c9de560dSAlex Tomas 2640c9de560dSAlex Tomas int ext4_mb_init(struct super_block *sb, int needs_recovery) 2641c9de560dSAlex Tomas { 2642c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 26436be2ded1SAneesh Kumar K.V unsigned i, j; 2644c9de560dSAlex Tomas unsigned offset; 2645c9de560dSAlex Tomas unsigned max; 264674767c5aSShen Feng int ret; 2647c9de560dSAlex Tomas 26481927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); 2649c9de560dSAlex Tomas 2650c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2651c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2652c9de560dSAlex Tomas return -ENOMEM; 2653c9de560dSAlex Tomas } 2654ff7ef329SYasunori Goto 26551927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); 2656c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2657c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2658a7b19448SDan Carpenter kfree(sbi->s_mb_offsets); 2659c9de560dSAlex Tomas return -ENOMEM; 2660c9de560dSAlex Tomas } 2661c9de560dSAlex Tomas 2662c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2663c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2664c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2665c9de560dSAlex Tomas 2666c9de560dSAlex Tomas i = 1; 2667c9de560dSAlex Tomas offset = 0; 2668c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2669c9de560dSAlex Tomas do { 2670c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2671c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2672c9de560dSAlex Tomas offset += 1 << (sb->s_blocksize_bits - i); 2673c9de560dSAlex Tomas max = max >> 1; 2674c9de560dSAlex Tomas i++; 2675c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2676c9de560dSAlex Tomas 2677c9de560dSAlex Tomas /* init file for buddy data */ 267874767c5aSShen Feng ret = ext4_mb_init_backend(sb); 267974767c5aSShen Feng if (ret != 0) { 2680c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2681c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 268274767c5aSShen Feng return ret; 2683c9de560dSAlex Tomas } 2684c9de560dSAlex Tomas 2685c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2686c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2687c9de560dSAlex Tomas 2688c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2689c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2690c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2691c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2692c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 2693c9de560dSAlex Tomas sbi->s_mb_history_filter = EXT4_MB_HISTORY_DEFAULT; 2694c9de560dSAlex Tomas sbi->s_mb_group_prealloc = MB_DEFAULT_GROUP_PREALLOC; 2695c9de560dSAlex Tomas 2696730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 2697c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2698c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2699c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2700c9de560dSAlex Tomas return -ENOMEM; 2701c9de560dSAlex Tomas } 2702730c213cSEric Sandeen for_each_possible_cpu(i) { 2703c9de560dSAlex Tomas struct ext4_locality_group *lg; 2704730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 2705c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 27066be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 27076be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 2708c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2709c9de560dSAlex Tomas } 2710c9de560dSAlex Tomas 2711c9de560dSAlex Tomas ext4_mb_history_init(sb); 2712c9de560dSAlex Tomas 27130390131bSFrank Mayhar if (sbi->s_journal) 27143e624fc7STheodore Ts'o sbi->s_journal->j_commit_callback = release_blocks_on_commit; 27153e624fc7STheodore Ts'o 27164776004fSTheodore Ts'o printk(KERN_INFO "EXT4-fs: mballoc enabled\n"); 2717c9de560dSAlex Tomas return 0; 2718c9de560dSAlex Tomas } 2719c9de560dSAlex Tomas 2720955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 2721c9de560dSAlex Tomas static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2722c9de560dSAlex Tomas { 2723c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2724c9de560dSAlex Tomas struct list_head *cur, *tmp; 2725c9de560dSAlex Tomas int count = 0; 2726c9de560dSAlex Tomas 2727c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2728c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2729c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2730c9de560dSAlex Tomas count++; 2731688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 2732c9de560dSAlex Tomas } 2733c9de560dSAlex Tomas if (count) 27346ba495e9STheodore Ts'o mb_debug(1, "mballoc: %u PAs left\n", count); 2735c9de560dSAlex Tomas 2736c9de560dSAlex Tomas } 2737c9de560dSAlex Tomas 2738c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2739c9de560dSAlex Tomas { 27408df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2741c9de560dSAlex Tomas ext4_group_t i; 2742c9de560dSAlex Tomas int num_meta_group_infos; 2743c9de560dSAlex Tomas struct ext4_group_info *grinfo; 2744c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2745c9de560dSAlex Tomas 2746c9de560dSAlex Tomas if (sbi->s_group_info) { 27478df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2748c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2749c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2750c9de560dSAlex Tomas kfree(grinfo->bb_bitmap); 2751c9de560dSAlex Tomas #endif 2752c9de560dSAlex Tomas ext4_lock_group(sb, i); 2753c9de560dSAlex Tomas ext4_mb_cleanup_pa(grinfo); 2754c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2755c9de560dSAlex Tomas kfree(grinfo); 2756c9de560dSAlex Tomas } 27578df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 2758c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2759c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2760c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2761c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2762c9de560dSAlex Tomas kfree(sbi->s_group_info); 2763c9de560dSAlex Tomas } 2764c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2765c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2766c9de560dSAlex Tomas if (sbi->s_buddy_cache) 2767c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2768c9de560dSAlex Tomas if (sbi->s_mb_stats) { 2769c9de560dSAlex Tomas printk(KERN_INFO 2770c9de560dSAlex Tomas "EXT4-fs: mballoc: %u blocks %u reqs (%u success)\n", 2771c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2772c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2773c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 2774c9de560dSAlex Tomas printk(KERN_INFO 2775c9de560dSAlex Tomas "EXT4-fs: mballoc: %u extents scanned, %u goal hits, " 2776c9de560dSAlex Tomas "%u 2^N hits, %u breaks, %u lost\n", 2777c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 2778c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 2779c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 2780c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 2781c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 2782c9de560dSAlex Tomas printk(KERN_INFO 2783c9de560dSAlex Tomas "EXT4-fs: mballoc: %lu generated and it took %Lu\n", 2784c9de560dSAlex Tomas sbi->s_mb_buddies_generated++, 2785c9de560dSAlex Tomas sbi->s_mb_generation_time); 2786c9de560dSAlex Tomas printk(KERN_INFO 2787c9de560dSAlex Tomas "EXT4-fs: mballoc: %u preallocated, %u discarded\n", 2788c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 2789c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 2790c9de560dSAlex Tomas } 2791c9de560dSAlex Tomas 2792730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 2793c9de560dSAlex Tomas ext4_mb_history_release(sb); 2794c9de560dSAlex Tomas 2795c9de560dSAlex Tomas return 0; 2796c9de560dSAlex Tomas } 2797c9de560dSAlex Tomas 27983e624fc7STheodore Ts'o /* 27993e624fc7STheodore Ts'o * This function is called by the jbd2 layer once the commit has finished, 28003e624fc7STheodore Ts'o * so we know we can free the blocks that were released with that commit. 28013e624fc7STheodore Ts'o */ 28023e624fc7STheodore Ts'o static void release_blocks_on_commit(journal_t *journal, transaction_t *txn) 2803c9de560dSAlex Tomas { 28043e624fc7STheodore Ts'o struct super_block *sb = journal->j_private; 2805c9de560dSAlex Tomas struct ext4_buddy e4b; 2806c894058dSAneesh Kumar K.V struct ext4_group_info *db; 2807c894058dSAneesh Kumar K.V int err, count = 0, count2 = 0; 2808c894058dSAneesh Kumar K.V struct ext4_free_data *entry; 28098a0aba73STheodore Ts'o ext4_fsblk_t discard_block; 28103e624fc7STheodore Ts'o struct list_head *l, *ltmp; 2811c9de560dSAlex Tomas 28123e624fc7STheodore Ts'o list_for_each_safe(l, ltmp, &txn->t_private_list) { 28133e624fc7STheodore Ts'o entry = list_entry(l, struct ext4_free_data, list); 2814c9de560dSAlex Tomas 28156ba495e9STheodore Ts'o mb_debug(1, "gonna free %u blocks in group %u (0x%p):", 2816c894058dSAneesh Kumar K.V entry->count, entry->group, entry); 2817c9de560dSAlex Tomas 2818c894058dSAneesh Kumar K.V err = ext4_mb_load_buddy(sb, entry->group, &e4b); 2819c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 2820c9de560dSAlex Tomas BUG_ON(err != 0); 2821c9de560dSAlex Tomas 2822c894058dSAneesh Kumar K.V db = e4b.bd_info; 2823c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 2824c894058dSAneesh Kumar K.V count += entry->count; 2825c9de560dSAlex Tomas count2++; 2826c894058dSAneesh Kumar K.V ext4_lock_group(sb, entry->group); 2827c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 2828c894058dSAneesh Kumar K.V rb_erase(&entry->node, &(db->bb_free_root)); 2829c894058dSAneesh Kumar K.V mb_free_blocks(NULL, &e4b, entry->start_blk, entry->count); 2830c9de560dSAlex Tomas 2831c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 2832c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 2833c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 2834c894058dSAneesh Kumar K.V */ 2835c9de560dSAlex Tomas page_cache_release(e4b.bd_buddy_page); 2836c9de560dSAlex Tomas page_cache_release(e4b.bd_bitmap_page); 2837c894058dSAneesh Kumar K.V } 2838c894058dSAneesh Kumar K.V ext4_unlock_group(sb, entry->group); 28398a0aba73STheodore Ts'o discard_block = (ext4_fsblk_t) entry->group * EXT4_BLOCKS_PER_GROUP(sb) 28408a0aba73STheodore Ts'o + entry->start_blk 28418a0aba73STheodore Ts'o + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 28429bffad1eSTheodore Ts'o trace_ext4_discard_blocks(sb, (unsigned long long)discard_block, 2843ba80b101STheodore Ts'o entry->count); 28448a0aba73STheodore Ts'o sb_issue_discard(sb, discard_block, entry->count); 2845c9de560dSAlex Tomas 2846c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 2847c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 28483e624fc7STheodore Ts'o } 2849c9de560dSAlex Tomas 28506ba495e9STheodore Ts'o mb_debug(1, "freed %u blocks in %u structures\n", count, count2); 2851c9de560dSAlex Tomas } 2852c9de560dSAlex Tomas 28536ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 28546ba495e9STheodore Ts'o u8 mb_enable_debug __read_mostly; 28556ba495e9STheodore Ts'o 28566ba495e9STheodore Ts'o static struct dentry *debugfs_dir; 28576ba495e9STheodore Ts'o static struct dentry *debugfs_debug; 28586ba495e9STheodore Ts'o 28596ba495e9STheodore Ts'o static void __init ext4_create_debugfs_entry(void) 28606ba495e9STheodore Ts'o { 28616ba495e9STheodore Ts'o debugfs_dir = debugfs_create_dir("ext4", NULL); 28626ba495e9STheodore Ts'o if (debugfs_dir) 28636ba495e9STheodore Ts'o debugfs_debug = debugfs_create_u8("mballoc-debug", 28646ba495e9STheodore Ts'o S_IRUGO | S_IWUSR, 28656ba495e9STheodore Ts'o debugfs_dir, 28666ba495e9STheodore Ts'o &mb_enable_debug); 28676ba495e9STheodore Ts'o } 28686ba495e9STheodore Ts'o 28696ba495e9STheodore Ts'o static void ext4_remove_debugfs_entry(void) 28706ba495e9STheodore Ts'o { 28716ba495e9STheodore Ts'o debugfs_remove(debugfs_debug); 28726ba495e9STheodore Ts'o debugfs_remove(debugfs_dir); 28736ba495e9STheodore Ts'o } 28746ba495e9STheodore Ts'o 28756ba495e9STheodore Ts'o #else 28766ba495e9STheodore Ts'o 28776ba495e9STheodore Ts'o static void __init ext4_create_debugfs_entry(void) 28786ba495e9STheodore Ts'o { 28796ba495e9STheodore Ts'o } 28806ba495e9STheodore Ts'o 28816ba495e9STheodore Ts'o static void ext4_remove_debugfs_entry(void) 28826ba495e9STheodore Ts'o { 28836ba495e9STheodore Ts'o } 28846ba495e9STheodore Ts'o 28856ba495e9STheodore Ts'o #endif 28866ba495e9STheodore Ts'o 2887c9de560dSAlex Tomas int __init init_ext4_mballoc(void) 2888c9de560dSAlex Tomas { 2889c9de560dSAlex Tomas ext4_pspace_cachep = 2890c9de560dSAlex Tomas kmem_cache_create("ext4_prealloc_space", 2891c9de560dSAlex Tomas sizeof(struct ext4_prealloc_space), 2892c9de560dSAlex Tomas 0, SLAB_RECLAIM_ACCOUNT, NULL); 2893c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 2894c9de560dSAlex Tomas return -ENOMEM; 2895c9de560dSAlex Tomas 2896256bdb49SEric Sandeen ext4_ac_cachep = 2897256bdb49SEric Sandeen kmem_cache_create("ext4_alloc_context", 2898256bdb49SEric Sandeen sizeof(struct ext4_allocation_context), 2899256bdb49SEric Sandeen 0, SLAB_RECLAIM_ACCOUNT, NULL); 2900256bdb49SEric Sandeen if (ext4_ac_cachep == NULL) { 2901256bdb49SEric Sandeen kmem_cache_destroy(ext4_pspace_cachep); 2902256bdb49SEric Sandeen return -ENOMEM; 2903256bdb49SEric Sandeen } 2904c894058dSAneesh Kumar K.V 2905c894058dSAneesh Kumar K.V ext4_free_ext_cachep = 2906c894058dSAneesh Kumar K.V kmem_cache_create("ext4_free_block_extents", 2907c894058dSAneesh Kumar K.V sizeof(struct ext4_free_data), 2908c894058dSAneesh Kumar K.V 0, SLAB_RECLAIM_ACCOUNT, NULL); 2909c894058dSAneesh Kumar K.V if (ext4_free_ext_cachep == NULL) { 2910c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_pspace_cachep); 2911c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_ac_cachep); 2912c894058dSAneesh Kumar K.V return -ENOMEM; 2913c894058dSAneesh Kumar K.V } 29146ba495e9STheodore Ts'o ext4_create_debugfs_entry(); 2915c9de560dSAlex Tomas return 0; 2916c9de560dSAlex Tomas } 2917c9de560dSAlex Tomas 2918c9de560dSAlex Tomas void exit_ext4_mballoc(void) 2919c9de560dSAlex Tomas { 29203e03f9caSJesper Dangaard Brouer /* 29213e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 29223e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 29233e03f9caSJesper Dangaard Brouer */ 29243e03f9caSJesper Dangaard Brouer rcu_barrier(); 2925c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 2926256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 2927c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_free_ext_cachep); 29286ba495e9STheodore Ts'o ext4_remove_debugfs_entry(); 2929c9de560dSAlex Tomas } 2930c9de560dSAlex Tomas 2931c9de560dSAlex Tomas 2932c9de560dSAlex Tomas /* 2933c9de560dSAlex Tomas * Check quota and mark choosed space (ac->ac_b_ex) non-free in bitmaps 2934c9de560dSAlex Tomas * Returns 0 if success or error code 2935c9de560dSAlex Tomas */ 29364ddfef7bSEric Sandeen static noinline_for_stack int 29374ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 2938498e5f24STheodore Ts'o handle_t *handle, unsigned int reserv_blks) 2939c9de560dSAlex Tomas { 2940c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 2941c9de560dSAlex Tomas struct ext4_super_block *es; 2942c9de560dSAlex Tomas struct ext4_group_desc *gdp; 2943c9de560dSAlex Tomas struct buffer_head *gdp_bh; 2944c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2945c9de560dSAlex Tomas struct super_block *sb; 2946c9de560dSAlex Tomas ext4_fsblk_t block; 2947519deca0SAneesh Kumar K.V int err, len; 2948c9de560dSAlex Tomas 2949c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 2950c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 2951c9de560dSAlex Tomas 2952c9de560dSAlex Tomas sb = ac->ac_sb; 2953c9de560dSAlex Tomas sbi = EXT4_SB(sb); 2954c9de560dSAlex Tomas es = sbi->s_es; 2955c9de560dSAlex Tomas 2956c9de560dSAlex Tomas 2957c9de560dSAlex Tomas err = -EIO; 2958574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 2959c9de560dSAlex Tomas if (!bitmap_bh) 2960c9de560dSAlex Tomas goto out_err; 2961c9de560dSAlex Tomas 2962c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 2963c9de560dSAlex Tomas if (err) 2964c9de560dSAlex Tomas goto out_err; 2965c9de560dSAlex Tomas 2966c9de560dSAlex Tomas err = -EIO; 2967c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 2968c9de560dSAlex Tomas if (!gdp) 2969c9de560dSAlex Tomas goto out_err; 2970c9de560dSAlex Tomas 2971a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 29729fd9784cSThadeu Lima de Souza Cascardo ext4_free_blks_count(sb, gdp)); 297303cddb80SAneesh Kumar K.V 2974c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 2975c9de560dSAlex Tomas if (err) 2976c9de560dSAlex Tomas goto out_err; 2977c9de560dSAlex Tomas 2978c9de560dSAlex Tomas block = ac->ac_b_ex.fe_group * EXT4_BLOCKS_PER_GROUP(sb) 2979c9de560dSAlex Tomas + ac->ac_b_ex.fe_start 2980c9de560dSAlex Tomas + le32_to_cpu(es->s_first_data_block); 2981c9de560dSAlex Tomas 2982519deca0SAneesh Kumar K.V len = ac->ac_b_ex.fe_len; 29836fd058f7STheodore Ts'o if (!ext4_data_block_valid(sbi, block, len)) { 298446e665e9SHarvey Harrison ext4_error(sb, __func__, 29856fd058f7STheodore Ts'o "Allocating blocks %llu-%llu which overlap " 29866fd058f7STheodore Ts'o "fs metadata\n", block, block+len); 2987519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 2988519deca0SAneesh Kumar K.V * Fix the bitmap and repeat the block allocation 2989519deca0SAneesh Kumar K.V * We leak some of the blocks here. 2990519deca0SAneesh Kumar K.V */ 2991955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 2992955ce5f5SAneesh Kumar K.V mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 2993519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 2994955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 29950390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 2996519deca0SAneesh Kumar K.V if (!err) 2997519deca0SAneesh Kumar K.V err = -EAGAIN; 2998519deca0SAneesh Kumar K.V goto out_err; 2999c9de560dSAlex Tomas } 3000955ce5f5SAneesh Kumar K.V 3001955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3002c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 3003c9de560dSAlex Tomas { 3004c9de560dSAlex Tomas int i; 3005c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 3006c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 3007c9de560dSAlex Tomas bitmap_bh->b_data)); 3008c9de560dSAlex Tomas } 3009c9de560dSAlex Tomas } 3010c9de560dSAlex Tomas #endif 3011955ce5f5SAneesh Kumar K.V mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,ac->ac_b_ex.fe_len); 3012c9de560dSAlex Tomas if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 3013c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 3014560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, 3015560671a0SAneesh Kumar K.V ext4_free_blocks_after_init(sb, 3016560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 3017c9de560dSAlex Tomas } 3018560671a0SAneesh Kumar K.V len = ext4_free_blks_count(sb, gdp) - ac->ac_b_ex.fe_len; 3019560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, len); 3020c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); 3021955ce5f5SAneesh Kumar K.V 3022955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 30236bc6e63fSAneesh Kumar K.V percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len); 3024d2a17637SMingming Cao /* 30256bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 3026d2a17637SMingming Cao */ 30276bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 30286bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 30296bc6e63fSAneesh Kumar K.V percpu_counter_sub(&sbi->s_dirtyblocks_counter, reserv_blks); 303060e58e0fSMingming Cao else { 30316bc6e63fSAneesh Kumar K.V percpu_counter_sub(&sbi->s_dirtyblocks_counter, 30326bc6e63fSAneesh Kumar K.V ac->ac_b_ex.fe_len); 303360e58e0fSMingming Cao /* convert reserved quota blocks to real quota blocks */ 303460e58e0fSMingming Cao vfs_dq_claim_block(ac->ac_inode, ac->ac_b_ex.fe_len); 303560e58e0fSMingming Cao } 3036c9de560dSAlex Tomas 3037772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 3038772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 3039772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 30409f24e420STheodore Ts'o atomic_sub(ac->ac_b_ex.fe_len, 30419f24e420STheodore Ts'o &sbi->s_flex_groups[flex_group].free_blocks); 3042772cb7c8SJose R. Santos } 3043772cb7c8SJose R. Santos 30440390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 3045c9de560dSAlex Tomas if (err) 3046c9de560dSAlex Tomas goto out_err; 30470390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 3048c9de560dSAlex Tomas 3049c9de560dSAlex Tomas out_err: 3050c9de560dSAlex Tomas sb->s_dirt = 1; 305142a10addSAneesh Kumar K.V brelse(bitmap_bh); 3052c9de560dSAlex Tomas return err; 3053c9de560dSAlex Tomas } 3054c9de560dSAlex Tomas 3055c9de560dSAlex Tomas /* 3056c9de560dSAlex Tomas * here we normalize request for locality group 3057c9de560dSAlex Tomas * Group request are normalized to s_strip size if we set the same via mount 3058c9de560dSAlex Tomas * option. If not we set it to s_mb_group_prealloc which can be configured via 3059b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 3060c9de560dSAlex Tomas * 3061c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 3062c9de560dSAlex Tomas */ 3063c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 3064c9de560dSAlex Tomas { 3065c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3066c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 3067c9de560dSAlex Tomas 3068c9de560dSAlex Tomas BUG_ON(lg == NULL); 3069c9de560dSAlex Tomas if (EXT4_SB(sb)->s_stripe) 3070c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_stripe; 3071c9de560dSAlex Tomas else 3072c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 30736ba495e9STheodore Ts'o mb_debug(1, "#%u: goal %u blocks for locality group\n", 3074c9de560dSAlex Tomas current->pid, ac->ac_g_ex.fe_len); 3075c9de560dSAlex Tomas } 3076c9de560dSAlex Tomas 3077c9de560dSAlex Tomas /* 3078c9de560dSAlex Tomas * Normalization means making request better in terms of 3079c9de560dSAlex Tomas * size and alignment 3080c9de560dSAlex Tomas */ 30814ddfef7bSEric Sandeen static noinline_for_stack void 30824ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 3083c9de560dSAlex Tomas struct ext4_allocation_request *ar) 3084c9de560dSAlex Tomas { 3085c9de560dSAlex Tomas int bsbits, max; 3086c9de560dSAlex Tomas ext4_lblk_t end; 3087c9de560dSAlex Tomas loff_t size, orig_size, start_off; 3088c9de560dSAlex Tomas ext4_lblk_t start, orig_start; 3089c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 30909a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 3091c9de560dSAlex Tomas 3092c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 3093c9de560dSAlex Tomas do not need preallocation */ 3094c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3095c9de560dSAlex Tomas return; 3096c9de560dSAlex Tomas 3097c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3098c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3099c9de560dSAlex Tomas return; 3100c9de560dSAlex Tomas 3101c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3102c9de560dSAlex Tomas * required (it's a tail, for example) */ 3103c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3104c9de560dSAlex Tomas return; 3105c9de560dSAlex Tomas 3106c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3107c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3108c9de560dSAlex Tomas return ; 3109c9de560dSAlex Tomas } 3110c9de560dSAlex Tomas 3111c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3112c9de560dSAlex Tomas 3113c9de560dSAlex Tomas /* first, let's learn actual file size 3114c9de560dSAlex Tomas * given current request is allocated */ 3115c9de560dSAlex Tomas size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len; 3116c9de560dSAlex Tomas size = size << bsbits; 3117c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3118c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 3119c9de560dSAlex Tomas 31201930479cSValerie Clement /* max size of free chunks */ 31211930479cSValerie Clement max = 2 << bsbits; 3122c9de560dSAlex Tomas 31231930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 31241930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 3125c9de560dSAlex Tomas 3126c9de560dSAlex Tomas /* first, try to predict filesize */ 3127c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3128c9de560dSAlex Tomas start_off = 0; 3129c9de560dSAlex Tomas if (size <= 16 * 1024) { 3130c9de560dSAlex Tomas size = 16 * 1024; 3131c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3132c9de560dSAlex Tomas size = 32 * 1024; 3133c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3134c9de560dSAlex Tomas size = 64 * 1024; 3135c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3136c9de560dSAlex Tomas size = 128 * 1024; 3137c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3138c9de560dSAlex Tomas size = 256 * 1024; 3139c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3140c9de560dSAlex Tomas size = 512 * 1024; 3141c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3142c9de560dSAlex Tomas size = 1024 * 1024; 31431930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 3144c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 31451930479cSValerie Clement (21 - bsbits)) << 21; 31461930479cSValerie Clement size = 2 * 1024 * 1024; 31471930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 3148c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3149c9de560dSAlex Tomas (22 - bsbits)) << 22; 3150c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3151c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 31521930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 3153c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3154c9de560dSAlex Tomas (23 - bsbits)) << 23; 3155c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3156c9de560dSAlex Tomas } else { 3157c9de560dSAlex Tomas start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits; 3158c9de560dSAlex Tomas size = ac->ac_o_ex.fe_len << bsbits; 3159c9de560dSAlex Tomas } 3160c9de560dSAlex Tomas orig_size = size = size >> bsbits; 3161c9de560dSAlex Tomas orig_start = start = start_off >> bsbits; 3162c9de560dSAlex Tomas 3163c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 3164c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 3165c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 3166c9de560dSAlex Tomas start = ar->lleft + 1; 3167c9de560dSAlex Tomas } 3168c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 3169c9de560dSAlex Tomas size -= start + size - ar->lright; 3170c9de560dSAlex Tomas 3171c9de560dSAlex Tomas end = start + size; 3172c9de560dSAlex Tomas 3173c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 3174c9de560dSAlex Tomas rcu_read_lock(); 31759a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3176498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3177c9de560dSAlex Tomas 3178c9de560dSAlex Tomas if (pa->pa_deleted) 3179c9de560dSAlex Tomas continue; 3180c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3181c9de560dSAlex Tomas if (pa->pa_deleted) { 3182c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3183c9de560dSAlex Tomas continue; 3184c9de560dSAlex Tomas } 3185c9de560dSAlex Tomas 3186c9de560dSAlex Tomas pa_end = pa->pa_lstart + pa->pa_len; 3187c9de560dSAlex Tomas 3188c9de560dSAlex Tomas /* PA must not overlap original request */ 3189c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 3190c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 3191c9de560dSAlex Tomas 319238877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 319338877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 3194c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3195c9de560dSAlex Tomas continue; 3196c9de560dSAlex Tomas } 3197c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3198c9de560dSAlex Tomas 319938877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 3200c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3201c9de560dSAlex Tomas BUG_ON(pa_end < start); 3202c9de560dSAlex Tomas start = pa_end; 320338877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3204c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3205c9de560dSAlex Tomas end = pa->pa_lstart; 3206c9de560dSAlex Tomas } 3207c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3208c9de560dSAlex Tomas } 3209c9de560dSAlex Tomas rcu_read_unlock(); 3210c9de560dSAlex Tomas size = end - start; 3211c9de560dSAlex Tomas 3212c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3213c9de560dSAlex Tomas rcu_read_lock(); 32149a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3215498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3216c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3217c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3218c9de560dSAlex Tomas pa_end = pa->pa_lstart + pa->pa_len; 3219c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3220c9de560dSAlex Tomas } 3221c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3222c9de560dSAlex Tomas } 3223c9de560dSAlex Tomas rcu_read_unlock(); 3224c9de560dSAlex Tomas 3225c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3226c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 3227c9de560dSAlex Tomas printk(KERN_ERR "start %lu, size %lu, fe_logical %lu\n", 3228c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3229c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3230c9de560dSAlex Tomas } 3231c9de560dSAlex Tomas BUG_ON(start + size <= ac->ac_o_ex.fe_logical && 3232c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical); 32338d03c7a0SEric Sandeen BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 3234c9de560dSAlex Tomas 3235c9de560dSAlex Tomas /* now prepare goal request */ 3236c9de560dSAlex Tomas 3237c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3238c9de560dSAlex Tomas * placement or satisfy big request as is */ 3239c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 3240c9de560dSAlex Tomas ac->ac_g_ex.fe_len = size; 3241c9de560dSAlex Tomas 3242c9de560dSAlex Tomas /* define goal start in order to merge */ 3243c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3244c9de560dSAlex Tomas /* merge to the right */ 3245c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3246c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3247c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3248c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3249c9de560dSAlex Tomas } 3250c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3251c9de560dSAlex Tomas /* merge to the left */ 3252c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3253c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3254c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3255c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3256c9de560dSAlex Tomas } 3257c9de560dSAlex Tomas 32586ba495e9STheodore Ts'o mb_debug(1, "goal: %u(was %u) blocks at %u\n", (unsigned) size, 3259c9de560dSAlex Tomas (unsigned) orig_size, (unsigned) start); 3260c9de560dSAlex Tomas } 3261c9de560dSAlex Tomas 3262c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3263c9de560dSAlex Tomas { 3264c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3265c9de560dSAlex Tomas 3266c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3267c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3268c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3269c9de560dSAlex Tomas if (ac->ac_o_ex.fe_len >= ac->ac_g_ex.fe_len) 3270c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3271c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3272c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3273c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3274c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3275c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3276c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3277c9de560dSAlex Tomas } 3278c9de560dSAlex Tomas 3279c9de560dSAlex Tomas ext4_mb_store_history(ac); 3280c9de560dSAlex Tomas } 3281c9de560dSAlex Tomas 3282c9de560dSAlex Tomas /* 3283c9de560dSAlex Tomas * use blocks preallocated to inode 3284c9de560dSAlex Tomas */ 3285c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3286c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3287c9de560dSAlex Tomas { 3288c9de560dSAlex Tomas ext4_fsblk_t start; 3289c9de560dSAlex Tomas ext4_fsblk_t end; 3290c9de560dSAlex Tomas int len; 3291c9de560dSAlex Tomas 3292c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3293c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 3294c9de560dSAlex Tomas end = min(pa->pa_pstart + pa->pa_len, start + ac->ac_o_ex.fe_len); 3295c9de560dSAlex Tomas len = end - start; 3296c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3297c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3298c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3299c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3300c9de560dSAlex Tomas ac->ac_pa = pa; 3301c9de560dSAlex Tomas 3302c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 3303c9de560dSAlex Tomas BUG_ON(start + len > pa->pa_pstart + pa->pa_len); 3304c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3305c9de560dSAlex Tomas pa->pa_free -= len; 3306c9de560dSAlex Tomas 33076ba495e9STheodore Ts'o mb_debug(1, "use %llu/%u from inode pa %p\n", start, len, pa); 3308c9de560dSAlex Tomas } 3309c9de560dSAlex Tomas 3310c9de560dSAlex Tomas /* 3311c9de560dSAlex Tomas * use blocks preallocated to locality group 3312c9de560dSAlex Tomas */ 3313c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3314c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3315c9de560dSAlex Tomas { 331603cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 33176be2ded1SAneesh Kumar K.V 3318c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3319c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3320c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3321c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3322c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3323c9de560dSAlex Tomas ac->ac_pa = pa; 3324c9de560dSAlex Tomas 3325c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 332626346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3327c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 332826346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 332926346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3330c9de560dSAlex Tomas */ 33316ba495e9STheodore Ts'o mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa); 3332c9de560dSAlex Tomas } 3333c9de560dSAlex Tomas 3334c9de560dSAlex Tomas /* 33355e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 33365e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 33375e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 33385e745b04SAneesh Kumar K.V * from the goal block. 33395e745b04SAneesh Kumar K.V */ 33405e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 33415e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 33425e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 33435e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 33445e745b04SAneesh Kumar K.V { 33455e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 33465e745b04SAneesh Kumar K.V 33475e745b04SAneesh Kumar K.V if (cpa == NULL) { 33485e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 33495e745b04SAneesh Kumar K.V return pa; 33505e745b04SAneesh Kumar K.V } 33515e745b04SAneesh Kumar K.V cur_distance = abs(goal_block - cpa->pa_pstart); 33525e745b04SAneesh Kumar K.V new_distance = abs(goal_block - pa->pa_pstart); 33535e745b04SAneesh Kumar K.V 33545e745b04SAneesh Kumar K.V if (cur_distance < new_distance) 33555e745b04SAneesh Kumar K.V return cpa; 33565e745b04SAneesh Kumar K.V 33575e745b04SAneesh Kumar K.V /* drop the previous reference */ 33585e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 33595e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 33605e745b04SAneesh Kumar K.V return pa; 33615e745b04SAneesh Kumar K.V } 33625e745b04SAneesh Kumar K.V 33635e745b04SAneesh Kumar K.V /* 3364c9de560dSAlex Tomas * search goal blocks in preallocated space 3365c9de560dSAlex Tomas */ 33664ddfef7bSEric Sandeen static noinline_for_stack int 33674ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3368c9de560dSAlex Tomas { 33696be2ded1SAneesh Kumar K.V int order, i; 3370c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3371c9de560dSAlex Tomas struct ext4_locality_group *lg; 33725e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 33735e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 3374c9de560dSAlex Tomas 3375c9de560dSAlex Tomas /* only data can be preallocated */ 3376c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3377c9de560dSAlex Tomas return 0; 3378c9de560dSAlex Tomas 3379c9de560dSAlex Tomas /* first, try per-file preallocation */ 3380c9de560dSAlex Tomas rcu_read_lock(); 33819a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3382c9de560dSAlex Tomas 3383c9de560dSAlex Tomas /* all fields in this condition don't change, 3384c9de560dSAlex Tomas * so we can skip locking for them */ 3385c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 3386c9de560dSAlex Tomas ac->ac_o_ex.fe_logical >= pa->pa_lstart + pa->pa_len) 3387c9de560dSAlex Tomas continue; 3388c9de560dSAlex Tomas 3389fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 3390fb0a387dSEric Sandeen if (!(EXT4_I(ac->ac_inode)->i_flags & EXT4_EXTENTS_FL) && 3391fb0a387dSEric Sandeen pa->pa_pstart + pa->pa_len > EXT4_MAX_BLOCK_FILE_PHYS) 3392fb0a387dSEric Sandeen continue; 3393fb0a387dSEric Sandeen 3394c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3395c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3396c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3397c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3398c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3399c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3400c9de560dSAlex Tomas ac->ac_criteria = 10; 3401c9de560dSAlex Tomas rcu_read_unlock(); 3402c9de560dSAlex Tomas return 1; 3403c9de560dSAlex Tomas } 3404c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3405c9de560dSAlex Tomas } 3406c9de560dSAlex Tomas rcu_read_unlock(); 3407c9de560dSAlex Tomas 3408c9de560dSAlex Tomas /* can we use group allocation? */ 3409c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 3410c9de560dSAlex Tomas return 0; 3411c9de560dSAlex Tomas 3412c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3413c9de560dSAlex Tomas lg = ac->ac_lg; 3414c9de560dSAlex Tomas if (lg == NULL) 3415c9de560dSAlex Tomas return 0; 34166be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 34176be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 34186be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 34196be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 3420c9de560dSAlex Tomas 34215e745b04SAneesh Kumar K.V goal_block = ac->ac_g_ex.fe_group * EXT4_BLOCKS_PER_GROUP(ac->ac_sb) + 34225e745b04SAneesh Kumar K.V ac->ac_g_ex.fe_start + 34235e745b04SAneesh Kumar K.V le32_to_cpu(EXT4_SB(ac->ac_sb)->s_es->s_first_data_block); 34245e745b04SAneesh Kumar K.V /* 34255e745b04SAneesh Kumar K.V * search for the prealloc space that is having 34265e745b04SAneesh Kumar K.V * minimal distance from the goal block. 34275e745b04SAneesh Kumar K.V */ 34286be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 3429c9de560dSAlex Tomas rcu_read_lock(); 34306be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 34316be2ded1SAneesh Kumar K.V pa_inode_list) { 3432c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 34336be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 34346be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 34355e745b04SAneesh Kumar K.V 34365e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 34375e745b04SAneesh Kumar K.V pa, cpa); 34385e745b04SAneesh Kumar K.V } 3439c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 34405e745b04SAneesh Kumar K.V } 34415e745b04SAneesh Kumar K.V rcu_read_unlock(); 34425e745b04SAneesh Kumar K.V } 34435e745b04SAneesh Kumar K.V if (cpa) { 34445e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 3445c9de560dSAlex Tomas ac->ac_criteria = 20; 3446c9de560dSAlex Tomas return 1; 3447c9de560dSAlex Tomas } 3448c9de560dSAlex Tomas return 0; 3449c9de560dSAlex Tomas } 3450c9de560dSAlex Tomas 3451c9de560dSAlex Tomas /* 34527a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 34537a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 34547a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 3455955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 34567a2fcbf7SAneesh Kumar K.V */ 34577a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 34587a2fcbf7SAneesh Kumar K.V ext4_group_t group) 34597a2fcbf7SAneesh Kumar K.V { 34607a2fcbf7SAneesh Kumar K.V struct rb_node *n; 34617a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 34627a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 34637a2fcbf7SAneesh Kumar K.V 34647a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 34657a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 34667a2fcbf7SAneesh Kumar K.V 34677a2fcbf7SAneesh Kumar K.V while (n) { 34687a2fcbf7SAneesh Kumar K.V entry = rb_entry(n, struct ext4_free_data, node); 3469955ce5f5SAneesh Kumar K.V mb_set_bits(bitmap, entry->start_blk, entry->count); 34707a2fcbf7SAneesh Kumar K.V n = rb_next(n); 34717a2fcbf7SAneesh Kumar K.V } 34727a2fcbf7SAneesh Kumar K.V return; 34737a2fcbf7SAneesh Kumar K.V } 34747a2fcbf7SAneesh Kumar K.V 34757a2fcbf7SAneesh Kumar K.V /* 3476c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3477c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3478955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 3479c9de560dSAlex Tomas */ 3480089ceeccSEric Sandeen static noinline_for_stack 3481089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3482c9de560dSAlex Tomas ext4_group_t group) 3483c9de560dSAlex Tomas { 3484c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3485c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3486c9de560dSAlex Tomas struct list_head *cur; 3487c9de560dSAlex Tomas ext4_group_t groupnr; 3488c9de560dSAlex Tomas ext4_grpblk_t start; 3489c9de560dSAlex Tomas int preallocated = 0; 3490c9de560dSAlex Tomas int count = 0; 3491c9de560dSAlex Tomas int len; 3492c9de560dSAlex Tomas 3493c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3494c9de560dSAlex Tomas * so the only competing code is preallocation use. 3495c9de560dSAlex Tomas * we don't need any locking here 3496c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3497c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3498c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3499c9de560dSAlex Tomas * is dropping preallocation 3500c9de560dSAlex Tomas */ 3501c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3502c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3503c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3504c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3505c9de560dSAlex Tomas &groupnr, &start); 3506c9de560dSAlex Tomas len = pa->pa_len; 3507c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3508c9de560dSAlex Tomas if (unlikely(len == 0)) 3509c9de560dSAlex Tomas continue; 3510c9de560dSAlex Tomas BUG_ON(groupnr != group); 3511955ce5f5SAneesh Kumar K.V mb_set_bits(bitmap, start, len); 3512c9de560dSAlex Tomas preallocated += len; 3513c9de560dSAlex Tomas count++; 3514c9de560dSAlex Tomas } 35156ba495e9STheodore Ts'o mb_debug(1, "prellocated %u for group %u\n", preallocated, group); 3516c9de560dSAlex Tomas } 3517c9de560dSAlex Tomas 3518c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3519c9de560dSAlex Tomas { 3520c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3521c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 3522c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3523c9de560dSAlex Tomas } 3524c9de560dSAlex Tomas 3525c9de560dSAlex Tomas /* 3526c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3527c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3528c9de560dSAlex Tomas */ 3529c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3530c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3531c9de560dSAlex Tomas { 3532a9df9a49STheodore Ts'o ext4_group_t grp; 3533d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 3534c9de560dSAlex Tomas 3535c9de560dSAlex Tomas if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) 3536c9de560dSAlex Tomas return; 3537c9de560dSAlex Tomas 3538c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3539c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3540c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3541c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3542c9de560dSAlex Tomas return; 3543c9de560dSAlex Tomas } 3544c9de560dSAlex Tomas 3545c9de560dSAlex Tomas pa->pa_deleted = 1; 3546c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3547c9de560dSAlex Tomas 3548d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 3549cc0fb9adSAneesh Kumar K.V /* 3550cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 3551cc0fb9adSAneesh Kumar K.V * next group when pa is used up 3552cc0fb9adSAneesh Kumar K.V */ 3553cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3554d33a1976SEric Sandeen grp_blk--; 3555d33a1976SEric Sandeen 3556d33a1976SEric Sandeen ext4_get_group_no_and_offset(sb, grp_blk, &grp, NULL); 3557c9de560dSAlex Tomas 3558c9de560dSAlex Tomas /* 3559c9de560dSAlex Tomas * possible race: 3560c9de560dSAlex Tomas * 3561c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3562c9de560dSAlex Tomas * find block B in PA 3563c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3564c9de560dSAlex Tomas * mark B in on-disk bitmap 3565c9de560dSAlex Tomas * drop PA from group 3566c9de560dSAlex Tomas * mark all PAs in buddy 3567c9de560dSAlex Tomas * 3568c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3569c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3570c9de560dSAlex Tomas * against that pair 3571c9de560dSAlex Tomas */ 3572c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3573c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3574c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3575c9de560dSAlex Tomas 3576c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3577c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3578c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3579c9de560dSAlex Tomas 3580c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3581c9de560dSAlex Tomas } 3582c9de560dSAlex Tomas 3583c9de560dSAlex Tomas /* 3584c9de560dSAlex Tomas * creates new preallocated space for given inode 3585c9de560dSAlex Tomas */ 35864ddfef7bSEric Sandeen static noinline_for_stack int 35874ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3588c9de560dSAlex Tomas { 3589c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3590c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3591c9de560dSAlex Tomas struct ext4_group_info *grp; 3592c9de560dSAlex Tomas struct ext4_inode_info *ei; 3593c9de560dSAlex Tomas 3594c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3595c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3596c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3597c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3598c9de560dSAlex Tomas 3599c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3600c9de560dSAlex Tomas if (pa == NULL) 3601c9de560dSAlex Tomas return -ENOMEM; 3602c9de560dSAlex Tomas 3603c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3604c9de560dSAlex Tomas int winl; 3605c9de560dSAlex Tomas int wins; 3606c9de560dSAlex Tomas int win; 3607c9de560dSAlex Tomas int offs; 3608c9de560dSAlex Tomas 3609c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3610c9de560dSAlex Tomas * so, found space must get proper lstart 3611c9de560dSAlex Tomas * to cover original request */ 3612c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3613c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3614c9de560dSAlex Tomas 3615c9de560dSAlex Tomas /* we're limited by original request in that 3616c9de560dSAlex Tomas * logical block must be covered any way 3617c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3618c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3619c9de560dSAlex Tomas 3620c9de560dSAlex Tomas /* also, we should cover whole original request */ 3621c9de560dSAlex Tomas wins = ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len; 3622c9de560dSAlex Tomas 3623c9de560dSAlex Tomas /* the smallest one defines real window */ 3624c9de560dSAlex Tomas win = min(winl, wins); 3625c9de560dSAlex Tomas 3626c9de560dSAlex Tomas offs = ac->ac_o_ex.fe_logical % ac->ac_b_ex.fe_len; 3627c9de560dSAlex Tomas if (offs && offs < win) 3628c9de560dSAlex Tomas win = offs; 3629c9de560dSAlex Tomas 3630c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - win; 3631c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3632c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3633c9de560dSAlex Tomas } 3634c9de560dSAlex Tomas 3635c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3636c9de560dSAlex Tomas * allocated blocks for history */ 3637c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3638c9de560dSAlex Tomas 3639c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3640c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3641c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3642c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3643c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3644c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3645d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3646d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3647c9de560dSAlex Tomas pa->pa_deleted = 0; 3648cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 3649c9de560dSAlex Tomas 36506ba495e9STheodore Ts'o mb_debug(1, "new inode pa %p: %llu/%u for %u\n", pa, 3651c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 36529bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 3653c9de560dSAlex Tomas 3654c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3655c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3656c9de560dSAlex Tomas 3657c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3658c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3659c9de560dSAlex Tomas 3660c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3661c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3662c9de560dSAlex Tomas 3663c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3664c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3665c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3666c9de560dSAlex Tomas 3667c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3668c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3669c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3670c9de560dSAlex Tomas 3671c9de560dSAlex Tomas return 0; 3672c9de560dSAlex Tomas } 3673c9de560dSAlex Tomas 3674c9de560dSAlex Tomas /* 3675c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3676c9de560dSAlex Tomas */ 36774ddfef7bSEric Sandeen static noinline_for_stack int 36784ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3679c9de560dSAlex Tomas { 3680c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3681c9de560dSAlex Tomas struct ext4_locality_group *lg; 3682c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3683c9de560dSAlex Tomas struct ext4_group_info *grp; 3684c9de560dSAlex Tomas 3685c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3686c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3687c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3688c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3689c9de560dSAlex Tomas 3690c9de560dSAlex Tomas BUG_ON(ext4_pspace_cachep == NULL); 3691c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3692c9de560dSAlex Tomas if (pa == NULL) 3693c9de560dSAlex Tomas return -ENOMEM; 3694c9de560dSAlex Tomas 3695c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3696c9de560dSAlex Tomas * allocated blocks for history */ 3697c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3698c9de560dSAlex Tomas 3699c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3700c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 3701c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3702c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3703c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3704c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 37056be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3706d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3707c9de560dSAlex Tomas pa->pa_deleted = 0; 3708cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 3709c9de560dSAlex Tomas 37106ba495e9STheodore Ts'o mb_debug(1, "new group pa %p: %llu/%u for %u\n", pa, 3711c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 37129bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 3713c9de560dSAlex Tomas 3714c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3715c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3716c9de560dSAlex Tomas 3717c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3718c9de560dSAlex Tomas lg = ac->ac_lg; 3719c9de560dSAlex Tomas BUG_ON(lg == NULL); 3720c9de560dSAlex Tomas 3721c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 3722c9de560dSAlex Tomas pa->pa_inode = NULL; 3723c9de560dSAlex Tomas 3724c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3725c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3726c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3727c9de560dSAlex Tomas 37286be2ded1SAneesh Kumar K.V /* 37296be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 37306be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 37316be2ded1SAneesh Kumar K.V */ 3732c9de560dSAlex Tomas return 0; 3733c9de560dSAlex Tomas } 3734c9de560dSAlex Tomas 3735c9de560dSAlex Tomas static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 3736c9de560dSAlex Tomas { 3737c9de560dSAlex Tomas int err; 3738c9de560dSAlex Tomas 3739c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 3740c9de560dSAlex Tomas err = ext4_mb_new_group_pa(ac); 3741c9de560dSAlex Tomas else 3742c9de560dSAlex Tomas err = ext4_mb_new_inode_pa(ac); 3743c9de560dSAlex Tomas return err; 3744c9de560dSAlex Tomas } 3745c9de560dSAlex Tomas 3746c9de560dSAlex Tomas /* 3747c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 3748c9de560dSAlex Tomas * in-core bitmap and buddy. 3749c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 3750c9de560dSAlex Tomas * nobody else can find/use it. 3751c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 3752c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 3753c9de560dSAlex Tomas */ 37544ddfef7bSEric Sandeen static noinline_for_stack int 37554ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 3756c83617dbSAneesh Kumar K.V struct ext4_prealloc_space *pa, 3757c83617dbSAneesh Kumar K.V struct ext4_allocation_context *ac) 3758c9de560dSAlex Tomas { 3759c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3760c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3761498e5f24STheodore Ts'o unsigned int end; 3762498e5f24STheodore Ts'o unsigned int next; 3763c9de560dSAlex Tomas ext4_group_t group; 3764c9de560dSAlex Tomas ext4_grpblk_t bit; 3765ba80b101STheodore Ts'o unsigned long long grp_blk_start; 3766c9de560dSAlex Tomas sector_t start; 3767c9de560dSAlex Tomas int err = 0; 3768c9de560dSAlex Tomas int free = 0; 3769c9de560dSAlex Tomas 3770c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3771c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3772ba80b101STheodore Ts'o grp_blk_start = pa->pa_pstart - bit; 3773c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3774c9de560dSAlex Tomas end = bit + pa->pa_len; 3775c9de560dSAlex Tomas 3776256bdb49SEric Sandeen if (ac) { 3777256bdb49SEric Sandeen ac->ac_sb = sb; 3778256bdb49SEric Sandeen ac->ac_inode = pa->pa_inode; 3779256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_DISCARD; 3780256bdb49SEric Sandeen } 3781c9de560dSAlex Tomas 3782c9de560dSAlex Tomas while (bit < end) { 3783ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 3784c9de560dSAlex Tomas if (bit >= end) 3785c9de560dSAlex Tomas break; 3786ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 3787c9de560dSAlex Tomas start = group * EXT4_BLOCKS_PER_GROUP(sb) + bit + 3788c9de560dSAlex Tomas le32_to_cpu(sbi->s_es->s_first_data_block); 37896ba495e9STheodore Ts'o mb_debug(1, " free preallocated %u/%u in group %u\n", 3790c9de560dSAlex Tomas (unsigned) start, (unsigned) next - bit, 3791c9de560dSAlex Tomas (unsigned) group); 3792c9de560dSAlex Tomas free += next - bit; 3793c9de560dSAlex Tomas 3794256bdb49SEric Sandeen if (ac) { 3795256bdb49SEric Sandeen ac->ac_b_ex.fe_group = group; 3796256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 3797256bdb49SEric Sandeen ac->ac_b_ex.fe_len = next - bit; 3798256bdb49SEric Sandeen ac->ac_b_ex.fe_logical = 0; 3799256bdb49SEric Sandeen ext4_mb_store_history(ac); 3800256bdb49SEric Sandeen } 3801c9de560dSAlex Tomas 38029bffad1eSTheodore Ts'o trace_ext4_mb_release_inode_pa(ac, pa, grp_blk_start + bit, 3803ba80b101STheodore Ts'o next - bit); 3804c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 3805c9de560dSAlex Tomas bit = next + 1; 3806c9de560dSAlex Tomas } 3807c9de560dSAlex Tomas if (free != pa->pa_free) { 380826346ff6SAneesh Kumar K.V printk(KERN_CRIT "pa %p: logic %lu, phys. %lu, len %lu\n", 3809c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 3810c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 3811c9de560dSAlex Tomas (unsigned long) pa->pa_len); 38125d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, group, 38135d1b1b3fSAneesh Kumar K.V __func__, "free %u, pa_free %u", 381426346ff6SAneesh Kumar K.V free, pa->pa_free); 3815e56eb659SAneesh Kumar K.V /* 3816e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 3817e56eb659SAneesh Kumar K.V * from the bitmap and continue. 3818e56eb659SAneesh Kumar K.V */ 3819c9de560dSAlex Tomas } 3820c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 3821c9de560dSAlex Tomas 3822c9de560dSAlex Tomas return err; 3823c9de560dSAlex Tomas } 3824c9de560dSAlex Tomas 38254ddfef7bSEric Sandeen static noinline_for_stack int 38264ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 3827c83617dbSAneesh Kumar K.V struct ext4_prealloc_space *pa, 3828c83617dbSAneesh Kumar K.V struct ext4_allocation_context *ac) 3829c9de560dSAlex Tomas { 3830c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3831c9de560dSAlex Tomas ext4_group_t group; 3832c9de560dSAlex Tomas ext4_grpblk_t bit; 3833c9de560dSAlex Tomas 3834256bdb49SEric Sandeen if (ac) 3835256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_DISCARD; 3836c9de560dSAlex Tomas 38379bffad1eSTheodore Ts'o trace_ext4_mb_release_group_pa(ac, pa); 3838c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3839c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3840c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3841c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 3842c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 3843c9de560dSAlex Tomas 3844256bdb49SEric Sandeen if (ac) { 3845256bdb49SEric Sandeen ac->ac_sb = sb; 3846256bdb49SEric Sandeen ac->ac_inode = NULL; 3847256bdb49SEric Sandeen ac->ac_b_ex.fe_group = group; 3848256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 3849256bdb49SEric Sandeen ac->ac_b_ex.fe_len = pa->pa_len; 3850256bdb49SEric Sandeen ac->ac_b_ex.fe_logical = 0; 3851256bdb49SEric Sandeen ext4_mb_store_history(ac); 3852256bdb49SEric Sandeen } 3853c9de560dSAlex Tomas 3854c9de560dSAlex Tomas return 0; 3855c9de560dSAlex Tomas } 3856c9de560dSAlex Tomas 3857c9de560dSAlex Tomas /* 3858c9de560dSAlex Tomas * releases all preallocations in given group 3859c9de560dSAlex Tomas * 3860c9de560dSAlex Tomas * first, we need to decide discard policy: 3861c9de560dSAlex Tomas * - when do we discard 3862c9de560dSAlex Tomas * 1) ENOSPC 3863c9de560dSAlex Tomas * - how many do we discard 3864c9de560dSAlex Tomas * 1) how many requested 3865c9de560dSAlex Tomas */ 38664ddfef7bSEric Sandeen static noinline_for_stack int 38674ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 3868c9de560dSAlex Tomas ext4_group_t group, int needed) 3869c9de560dSAlex Tomas { 3870c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3871c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3872c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3873c83617dbSAneesh Kumar K.V struct ext4_allocation_context *ac; 3874c9de560dSAlex Tomas struct list_head list; 3875c9de560dSAlex Tomas struct ext4_buddy e4b; 3876c9de560dSAlex Tomas int err; 3877c9de560dSAlex Tomas int busy = 0; 3878c9de560dSAlex Tomas int free = 0; 3879c9de560dSAlex Tomas 38806ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for group %u\n", group); 3881c9de560dSAlex Tomas 3882c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 3883c9de560dSAlex Tomas return 0; 3884c9de560dSAlex Tomas 3885574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 3886c9de560dSAlex Tomas if (bitmap_bh == NULL) { 3887ce89f46cSAneesh Kumar K.V ext4_error(sb, __func__, "Error in reading block " 3888a9df9a49STheodore Ts'o "bitmap for %u", group); 3889ce89f46cSAneesh Kumar K.V return 0; 3890c9de560dSAlex Tomas } 3891c9de560dSAlex Tomas 3892c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3893ce89f46cSAneesh Kumar K.V if (err) { 3894ce89f46cSAneesh Kumar K.V ext4_error(sb, __func__, "Error in loading buddy " 3895a9df9a49STheodore Ts'o "information for %u", group); 3896ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 3897ce89f46cSAneesh Kumar K.V return 0; 3898ce89f46cSAneesh Kumar K.V } 3899c9de560dSAlex Tomas 3900c9de560dSAlex Tomas if (needed == 0) 3901c9de560dSAlex Tomas needed = EXT4_BLOCKS_PER_GROUP(sb) + 1; 3902c9de560dSAlex Tomas 3903c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3904c83617dbSAneesh Kumar K.V ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 39059bffad1eSTheodore Ts'o if (ac) 39069bffad1eSTheodore Ts'o ac->ac_sb = sb; 3907c9de560dSAlex Tomas repeat: 3908c9de560dSAlex Tomas ext4_lock_group(sb, group); 3909c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 3910c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 3911c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3912c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3913c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3914c9de560dSAlex Tomas busy = 1; 3915c9de560dSAlex Tomas continue; 3916c9de560dSAlex Tomas } 3917c9de560dSAlex Tomas if (pa->pa_deleted) { 3918c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3919c9de560dSAlex Tomas continue; 3920c9de560dSAlex Tomas } 3921c9de560dSAlex Tomas 3922c9de560dSAlex Tomas /* seems this one can be freed ... */ 3923c9de560dSAlex Tomas pa->pa_deleted = 1; 3924c9de560dSAlex Tomas 3925c9de560dSAlex Tomas /* we can trust pa_free ... */ 3926c9de560dSAlex Tomas free += pa->pa_free; 3927c9de560dSAlex Tomas 3928c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3929c9de560dSAlex Tomas 3930c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3931c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3932c9de560dSAlex Tomas } 3933c9de560dSAlex Tomas 3934c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 3935c9de560dSAlex Tomas if (free < needed && busy) { 3936c9de560dSAlex Tomas busy = 0; 3937c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3938c9de560dSAlex Tomas /* 3939c9de560dSAlex Tomas * Yield the CPU here so that we don't get soft lockup 3940c9de560dSAlex Tomas * in non preempt case. 3941c9de560dSAlex Tomas */ 3942c9de560dSAlex Tomas yield(); 3943c9de560dSAlex Tomas goto repeat; 3944c9de560dSAlex Tomas } 3945c9de560dSAlex Tomas 3946c9de560dSAlex Tomas /* found anything to free? */ 3947c9de560dSAlex Tomas if (list_empty(&list)) { 3948c9de560dSAlex Tomas BUG_ON(free != 0); 3949c9de560dSAlex Tomas goto out; 3950c9de560dSAlex Tomas } 3951c9de560dSAlex Tomas 3952c9de560dSAlex Tomas /* now free all selected PAs */ 3953c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3954c9de560dSAlex Tomas 3955c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 3956c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3957c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3958c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3959c9de560dSAlex Tomas 3960cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3961c83617dbSAneesh Kumar K.V ext4_mb_release_group_pa(&e4b, pa, ac); 3962c9de560dSAlex Tomas else 3963c83617dbSAneesh Kumar K.V ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa, ac); 3964c9de560dSAlex Tomas 3965c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3966c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3967c9de560dSAlex Tomas } 3968c9de560dSAlex Tomas 3969c9de560dSAlex Tomas out: 3970c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3971c83617dbSAneesh Kumar K.V if (ac) 3972c83617dbSAneesh Kumar K.V kmem_cache_free(ext4_ac_cachep, ac); 3973c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 3974c9de560dSAlex Tomas put_bh(bitmap_bh); 3975c9de560dSAlex Tomas return free; 3976c9de560dSAlex Tomas } 3977c9de560dSAlex Tomas 3978c9de560dSAlex Tomas /* 3979c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 3980c9de560dSAlex Tomas * 3981c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 3982c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 3983c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 3984c9de560dSAlex Tomas * 3985c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 3986c9de560dSAlex Tomas */ 3987c2ea3fdeSTheodore Ts'o void ext4_discard_preallocations(struct inode *inode) 3988c9de560dSAlex Tomas { 3989c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 3990c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 3991c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3992c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3993c83617dbSAneesh Kumar K.V struct ext4_allocation_context *ac; 3994c9de560dSAlex Tomas ext4_group_t group = 0; 3995c9de560dSAlex Tomas struct list_head list; 3996c9de560dSAlex Tomas struct ext4_buddy e4b; 3997c9de560dSAlex Tomas int err; 3998c9de560dSAlex Tomas 3999c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 4000c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 4001c9de560dSAlex Tomas return; 4002c9de560dSAlex Tomas } 4003c9de560dSAlex Tomas 40046ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino); 40059bffad1eSTheodore Ts'o trace_ext4_discard_preallocations(inode); 4006c9de560dSAlex Tomas 4007c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 4008c9de560dSAlex Tomas 4009c83617dbSAneesh Kumar K.V ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 40109bffad1eSTheodore Ts'o if (ac) { 40119bffad1eSTheodore Ts'o ac->ac_sb = sb; 40129bffad1eSTheodore Ts'o ac->ac_inode = inode; 40139bffad1eSTheodore Ts'o } 4014c9de560dSAlex Tomas repeat: 4015c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 4016c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 4017c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 4018c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 4019c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 4020c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 4021c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4022c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 4023c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 4024c9de560dSAlex Tomas * use preallocation while we're discarding it */ 4025c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4026c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4027c9de560dSAlex Tomas printk(KERN_ERR "uh-oh! used pa while discarding\n"); 4028c9de560dSAlex Tomas WARN_ON(1); 4029c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4030c9de560dSAlex Tomas goto repeat; 4031c9de560dSAlex Tomas 4032c9de560dSAlex Tomas } 4033c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 4034c9de560dSAlex Tomas pa->pa_deleted = 1; 4035c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4036c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 4037c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 4038c9de560dSAlex Tomas continue; 4039c9de560dSAlex Tomas } 4040c9de560dSAlex Tomas 4041c9de560dSAlex Tomas /* someone is deleting pa right now */ 4042c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4043c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4044c9de560dSAlex Tomas 4045c9de560dSAlex Tomas /* we have to wait here because pa_deleted 4046c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 4047c9de560dSAlex Tomas * the list. as we might be called from 4048c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 4049c9de560dSAlex Tomas * and concurrent thread which is unlinking 4050c9de560dSAlex Tomas * pa from inode's list may access already 4051c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 4052c9de560dSAlex Tomas 4053c9de560dSAlex Tomas /* XXX: if this happens too often, we can 4054c9de560dSAlex Tomas * add a flag to force wait only in case 4055c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 4056c9de560dSAlex Tomas * regular truncate */ 4057c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4058c9de560dSAlex Tomas goto repeat; 4059c9de560dSAlex Tomas } 4060c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4061c9de560dSAlex Tomas 4062c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4063cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 4064c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL); 4065c9de560dSAlex Tomas 4066c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 4067ce89f46cSAneesh Kumar K.V if (err) { 4068ce89f46cSAneesh Kumar K.V ext4_error(sb, __func__, "Error in loading buddy " 4069a9df9a49STheodore Ts'o "information for %u", group); 4070ce89f46cSAneesh Kumar K.V continue; 4071ce89f46cSAneesh Kumar K.V } 4072c9de560dSAlex Tomas 4073574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 4074c9de560dSAlex Tomas if (bitmap_bh == NULL) { 4075ce89f46cSAneesh Kumar K.V ext4_error(sb, __func__, "Error in reading block " 4076a9df9a49STheodore Ts'o "bitmap for %u", group); 4077c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4078ce89f46cSAneesh Kumar K.V continue; 4079c9de560dSAlex Tomas } 4080c9de560dSAlex Tomas 4081c9de560dSAlex Tomas ext4_lock_group(sb, group); 4082c9de560dSAlex Tomas list_del(&pa->pa_group_list); 4083c83617dbSAneesh Kumar K.V ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa, ac); 4084c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4085c9de560dSAlex Tomas 4086c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4087c9de560dSAlex Tomas put_bh(bitmap_bh); 4088c9de560dSAlex Tomas 4089c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4090c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4091c9de560dSAlex Tomas } 4092c83617dbSAneesh Kumar K.V if (ac) 4093c83617dbSAneesh Kumar K.V kmem_cache_free(ext4_ac_cachep, ac); 4094c9de560dSAlex Tomas } 4095c9de560dSAlex Tomas 4096c9de560dSAlex Tomas /* 4097c9de560dSAlex Tomas * finds all preallocated spaces and return blocks being freed to them 4098c9de560dSAlex Tomas * if preallocated space becomes full (no block is used from the space) 4099c9de560dSAlex Tomas * then the function frees space in buddy 4100c9de560dSAlex Tomas * XXX: at the moment, truncate (which is the only way to free blocks) 4101c9de560dSAlex Tomas * discards all preallocations 4102c9de560dSAlex Tomas */ 4103c9de560dSAlex Tomas static void ext4_mb_return_to_preallocation(struct inode *inode, 4104c9de560dSAlex Tomas struct ext4_buddy *e4b, 4105c9de560dSAlex Tomas sector_t block, int count) 4106c9de560dSAlex Tomas { 4107c9de560dSAlex Tomas BUG_ON(!list_empty(&EXT4_I(inode)->i_prealloc_list)); 4108c9de560dSAlex Tomas } 41096ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 4110c9de560dSAlex Tomas static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4111c9de560dSAlex Tomas { 4112c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 41138df9675fSTheodore Ts'o ext4_group_t ngroups, i; 4114c9de560dSAlex Tomas 4115c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: Can't allocate:" 4116c9de560dSAlex Tomas " Allocation context details:\n"); 4117c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: status %d flags %d\n", 4118c9de560dSAlex Tomas ac->ac_status, ac->ac_flags); 4119c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: orig %lu/%lu/%lu@%lu, goal %lu/%lu/%lu@%lu, " 4120c9de560dSAlex Tomas "best %lu/%lu/%lu@%lu cr %d\n", 4121c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_group, 4122c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_start, 4123c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_len, 4124c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_logical, 4125c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_group, 4126c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_start, 4127c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_len, 4128c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_logical, 4129c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_group, 4130c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_start, 4131c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_len, 4132c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_logical, 4133c9de560dSAlex Tomas (int)ac->ac_criteria); 4134c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: %lu scanned, %d found\n", ac->ac_ex_scanned, 4135c9de560dSAlex Tomas ac->ac_found); 4136c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: groups: \n"); 41378df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 41388df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 4139c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 4140c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4141c9de560dSAlex Tomas ext4_grpblk_t start; 4142c9de560dSAlex Tomas struct list_head *cur; 4143c9de560dSAlex Tomas ext4_lock_group(sb, i); 4144c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4145c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 4146c9de560dSAlex Tomas pa_group_list); 4147c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4148c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4149c9de560dSAlex Tomas NULL, &start); 4150c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 41511c718505SAkira Fujita printk(KERN_ERR "PA:%u:%d:%u \n", i, 4152c9de560dSAlex Tomas start, pa->pa_len); 4153c9de560dSAlex Tomas } 415460bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 4155c9de560dSAlex Tomas 4156c9de560dSAlex Tomas if (grp->bb_free == 0) 4157c9de560dSAlex Tomas continue; 41581c718505SAkira Fujita printk(KERN_ERR "%u: %d/%d \n", 4159c9de560dSAlex Tomas i, grp->bb_free, grp->bb_fragments); 4160c9de560dSAlex Tomas } 4161c9de560dSAlex Tomas printk(KERN_ERR "\n"); 4162c9de560dSAlex Tomas } 4163c9de560dSAlex Tomas #else 4164c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4165c9de560dSAlex Tomas { 4166c9de560dSAlex Tomas return; 4167c9de560dSAlex Tomas } 4168c9de560dSAlex Tomas #endif 4169c9de560dSAlex Tomas 4170c9de560dSAlex Tomas /* 4171c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 4172c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 4173c9de560dSAlex Tomas * allocation which ever is larger 4174c9de560dSAlex Tomas * 4175b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 4176c9de560dSAlex Tomas */ 4177c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 4178c9de560dSAlex Tomas { 4179c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4180c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 4181c9de560dSAlex Tomas loff_t size, isize; 4182c9de560dSAlex Tomas 4183c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4184c9de560dSAlex Tomas return; 4185c9de560dSAlex Tomas 41864ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 41874ba74d00STheodore Ts'o return; 41884ba74d00STheodore Ts'o 4189c9de560dSAlex Tomas size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len; 419050797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 419150797481STheodore Ts'o >> bsbits; 4192c9de560dSAlex Tomas 419350797481STheodore Ts'o if ((size == isize) && 419450797481STheodore Ts'o !ext4_fs_is_busy(sbi) && 419550797481STheodore Ts'o (atomic_read(&ac->ac_inode->i_writecount) == 0)) { 419650797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 419750797481STheodore Ts'o return; 419850797481STheodore Ts'o } 419950797481STheodore Ts'o 4200c9de560dSAlex Tomas /* don't use group allocation for large files */ 420171780577STheodore Ts'o size = max(size, isize); 42024ba74d00STheodore Ts'o if (size >= sbi->s_mb_stream_request) { 42034ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 4204c9de560dSAlex Tomas return; 42054ba74d00STheodore Ts'o } 4206c9de560dSAlex Tomas 4207c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 4208c9de560dSAlex Tomas /* 4209c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 4210c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 4211c9de560dSAlex Tomas * request from multiple CPUs. 4212c9de560dSAlex Tomas */ 4213730c213cSEric Sandeen ac->ac_lg = per_cpu_ptr(sbi->s_locality_groups, raw_smp_processor_id()); 4214c9de560dSAlex Tomas 4215c9de560dSAlex Tomas /* we're going to use group allocation */ 4216c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4217c9de560dSAlex Tomas 4218c9de560dSAlex Tomas /* serialize all allocations in the group */ 4219c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4220c9de560dSAlex Tomas } 4221c9de560dSAlex Tomas 42224ddfef7bSEric Sandeen static noinline_for_stack int 42234ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4224c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4225c9de560dSAlex Tomas { 4226c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4227c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4228c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4229c9de560dSAlex Tomas ext4_group_t group; 4230498e5f24STheodore Ts'o unsigned int len; 4231498e5f24STheodore Ts'o ext4_fsblk_t goal; 4232c9de560dSAlex Tomas ext4_grpblk_t block; 4233c9de560dSAlex Tomas 4234c9de560dSAlex Tomas /* we can't allocate > group size */ 4235c9de560dSAlex Tomas len = ar->len; 4236c9de560dSAlex Tomas 4237c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 4238c9de560dSAlex Tomas if (len >= EXT4_BLOCKS_PER_GROUP(sb) - 10) 4239c9de560dSAlex Tomas len = EXT4_BLOCKS_PER_GROUP(sb) - 10; 4240c9de560dSAlex Tomas 4241c9de560dSAlex Tomas /* start searching from the goal */ 4242c9de560dSAlex Tomas goal = ar->goal; 4243c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4244c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4245c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4246c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4247c9de560dSAlex Tomas 4248c9de560dSAlex Tomas /* set up allocation goals */ 4249833576b3STheodore Ts'o memset(ac, 0, sizeof(struct ext4_allocation_context)); 4250c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ar->logical; 4251c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4252c9de560dSAlex Tomas ac->ac_sb = sb; 4253c9de560dSAlex Tomas ac->ac_inode = ar->inode; 4254c9de560dSAlex Tomas ac->ac_o_ex.fe_logical = ar->logical; 4255c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4256c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4257c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 4258c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = ar->logical; 4259c9de560dSAlex Tomas ac->ac_g_ex.fe_group = group; 4260c9de560dSAlex Tomas ac->ac_g_ex.fe_start = block; 4261c9de560dSAlex Tomas ac->ac_g_ex.fe_len = len; 4262c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4263c9de560dSAlex Tomas 4264c9de560dSAlex Tomas /* we have to define context: we'll we work with a file or 4265c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4266c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4267c9de560dSAlex Tomas 42686ba495e9STheodore Ts'o mb_debug(1, "init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, " 4269c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4270c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4271c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4272c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4273c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 4274c9de560dSAlex Tomas atomic_read(&ar->inode->i_writecount) ? "" : "non-"); 4275c9de560dSAlex Tomas return 0; 4276c9de560dSAlex Tomas 4277c9de560dSAlex Tomas } 4278c9de560dSAlex Tomas 42796be2ded1SAneesh Kumar K.V static noinline_for_stack void 42806be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 42816be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 42826be2ded1SAneesh Kumar K.V int order, int total_entries) 42836be2ded1SAneesh Kumar K.V { 42846be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 42856be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 42866be2ded1SAneesh Kumar K.V struct list_head discard_list; 42876be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 42886be2ded1SAneesh Kumar K.V struct ext4_allocation_context *ac; 42896be2ded1SAneesh Kumar K.V 42906ba495e9STheodore Ts'o mb_debug(1, "discard locality group preallocation\n"); 42916be2ded1SAneesh Kumar K.V 42926be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 42936be2ded1SAneesh Kumar K.V ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 42949bffad1eSTheodore Ts'o if (ac) 42959bffad1eSTheodore Ts'o ac->ac_sb = sb; 42966be2ded1SAneesh Kumar K.V 42976be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 42986be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 42996be2ded1SAneesh Kumar K.V pa_inode_list) { 43006be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 43016be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 43026be2ded1SAneesh Kumar K.V /* 43036be2ded1SAneesh Kumar K.V * This is the pa that we just used 43046be2ded1SAneesh Kumar K.V * for block allocation. So don't 43056be2ded1SAneesh Kumar K.V * free that 43066be2ded1SAneesh Kumar K.V */ 43076be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43086be2ded1SAneesh Kumar K.V continue; 43096be2ded1SAneesh Kumar K.V } 43106be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 43116be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43126be2ded1SAneesh Kumar K.V continue; 43136be2ded1SAneesh Kumar K.V } 43146be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 4315cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 43166be2ded1SAneesh Kumar K.V 43176be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 43186be2ded1SAneesh Kumar K.V pa->pa_deleted = 1; 43196be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 43206be2ded1SAneesh Kumar K.V 43216be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 43226be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 43236be2ded1SAneesh Kumar K.V 43246be2ded1SAneesh Kumar K.V total_entries--; 43256be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 43266be2ded1SAneesh Kumar K.V /* 43276be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 43286be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 43296be2ded1SAneesh Kumar K.V * mak sure we don't call discard 43306be2ded1SAneesh Kumar K.V * soon for this list. 43316be2ded1SAneesh Kumar K.V */ 43326be2ded1SAneesh Kumar K.V break; 43336be2ded1SAneesh Kumar K.V } 43346be2ded1SAneesh Kumar K.V } 43356be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 43366be2ded1SAneesh Kumar K.V 43376be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 43386be2ded1SAneesh Kumar K.V 43396be2ded1SAneesh Kumar K.V ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL); 43406be2ded1SAneesh Kumar K.V if (ext4_mb_load_buddy(sb, group, &e4b)) { 43416be2ded1SAneesh Kumar K.V ext4_error(sb, __func__, "Error in loading buddy " 4342a9df9a49STheodore Ts'o "information for %u", group); 43436be2ded1SAneesh Kumar K.V continue; 43446be2ded1SAneesh Kumar K.V } 43456be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 43466be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 43476be2ded1SAneesh Kumar K.V ext4_mb_release_group_pa(&e4b, pa, ac); 43486be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 43496be2ded1SAneesh Kumar K.V 43506be2ded1SAneesh Kumar K.V ext4_mb_release_desc(&e4b); 43516be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 43526be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 43536be2ded1SAneesh Kumar K.V } 43546be2ded1SAneesh Kumar K.V if (ac) 43556be2ded1SAneesh Kumar K.V kmem_cache_free(ext4_ac_cachep, ac); 43566be2ded1SAneesh Kumar K.V } 43576be2ded1SAneesh Kumar K.V 43586be2ded1SAneesh Kumar K.V /* 43596be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 43606be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 43616be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 43626be2ded1SAneesh Kumar K.V * 43636be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 43646be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 43656be2ded1SAneesh Kumar K.V */ 43666be2ded1SAneesh Kumar K.V 43676be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 43686be2ded1SAneesh Kumar K.V { 43696be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 43706be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 43716be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 43726be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 43736be2ded1SAneesh Kumar K.V 43746be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 43756be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 43766be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 43776be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 43786be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 43796be2ded1SAneesh Kumar K.V rcu_read_lock(); 43806be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 43816be2ded1SAneesh Kumar K.V pa_inode_list) { 43826be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 43836be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 4384e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 43856be2ded1SAneesh Kumar K.V continue; 43866be2ded1SAneesh Kumar K.V } 43876be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 43886be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 43896be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 43906be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 43916be2ded1SAneesh Kumar K.V added = 1; 43926be2ded1SAneesh Kumar K.V /* 43936be2ded1SAneesh Kumar K.V * we want to count the total 43946be2ded1SAneesh Kumar K.V * number of entries in the list 43956be2ded1SAneesh Kumar K.V */ 43966be2ded1SAneesh Kumar K.V } 43976be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 43986be2ded1SAneesh Kumar K.V lg_prealloc_count++; 43996be2ded1SAneesh Kumar K.V } 44006be2ded1SAneesh Kumar K.V if (!added) 44016be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 44026be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 44036be2ded1SAneesh Kumar K.V rcu_read_unlock(); 44046be2ded1SAneesh Kumar K.V 44056be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 44066be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 44076be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 44086be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 44096be2ded1SAneesh Kumar K.V return; 44106be2ded1SAneesh Kumar K.V } 44116be2ded1SAneesh Kumar K.V return ; 44126be2ded1SAneesh Kumar K.V } 44136be2ded1SAneesh Kumar K.V 4414c9de560dSAlex Tomas /* 4415c9de560dSAlex Tomas * release all resource we used in allocation 4416c9de560dSAlex Tomas */ 4417c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4418c9de560dSAlex Tomas { 44196be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 44206be2ded1SAneesh Kumar K.V if (pa) { 4421cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 4422c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 44236be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 44246be2ded1SAneesh Kumar K.V pa->pa_pstart += ac->ac_b_ex.fe_len; 44256be2ded1SAneesh Kumar K.V pa->pa_lstart += ac->ac_b_ex.fe_len; 44266be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 44276be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 44286be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 4429ba443916SAneesh Kumar K.V } 4430ba443916SAneesh Kumar K.V } 4431ba443916SAneesh Kumar K.V if (ac->alloc_semp) 4432ba443916SAneesh Kumar K.V up_read(ac->alloc_semp); 4433ba443916SAneesh Kumar K.V if (pa) { 44346be2ded1SAneesh Kumar K.V /* 44356be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 44366be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 44376be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 4438ba443916SAneesh Kumar K.V * doesn't grow big. We need to release 4439ba443916SAneesh Kumar K.V * alloc_semp before calling ext4_mb_add_n_trim() 44406be2ded1SAneesh Kumar K.V */ 4441cc0fb9adSAneesh Kumar K.V if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) { 44426be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 44436be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 44446be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 44456be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 4446c9de560dSAlex Tomas } 44476be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 4448c9de560dSAlex Tomas } 4449c9de560dSAlex Tomas if (ac->ac_bitmap_page) 4450c9de560dSAlex Tomas page_cache_release(ac->ac_bitmap_page); 4451c9de560dSAlex Tomas if (ac->ac_buddy_page) 4452c9de560dSAlex Tomas page_cache_release(ac->ac_buddy_page); 4453c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4454c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4455c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4456c9de560dSAlex Tomas return 0; 4457c9de560dSAlex Tomas } 4458c9de560dSAlex Tomas 4459c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4460c9de560dSAlex Tomas { 44618df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 4462c9de560dSAlex Tomas int ret; 4463c9de560dSAlex Tomas int freed = 0; 4464c9de560dSAlex Tomas 44659bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 44668df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 4467c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4468c9de560dSAlex Tomas freed += ret; 4469c9de560dSAlex Tomas needed -= ret; 4470c9de560dSAlex Tomas } 4471c9de560dSAlex Tomas 4472c9de560dSAlex Tomas return freed; 4473c9de560dSAlex Tomas } 4474c9de560dSAlex Tomas 4475c9de560dSAlex Tomas /* 4476c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4477c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4478c9de560dSAlex Tomas * to usual allocation 4479c9de560dSAlex Tomas */ 4480c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4481c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4482c9de560dSAlex Tomas { 44836bc6e63fSAneesh Kumar K.V int freed; 4484256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4485c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4486c9de560dSAlex Tomas struct super_block *sb; 4487c9de560dSAlex Tomas ext4_fsblk_t block = 0; 448860e58e0fSMingming Cao unsigned int inquota = 0; 4489498e5f24STheodore Ts'o unsigned int reserv_blks = 0; 4490c9de560dSAlex Tomas 4491c9de560dSAlex Tomas sb = ar->inode->i_sb; 4492c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4493c9de560dSAlex Tomas 44949bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 4495ba80b101STheodore Ts'o 4496d2a17637SMingming Cao /* 449760e58e0fSMingming Cao * For delayed allocation, we could skip the ENOSPC and 449860e58e0fSMingming Cao * EDQUOT check, as blocks and quotas have been already 449960e58e0fSMingming Cao * reserved when data being copied into pagecache. 450060e58e0fSMingming Cao */ 450160e58e0fSMingming Cao if (EXT4_I(ar->inode)->i_delalloc_reserved_flag) 450260e58e0fSMingming Cao ar->flags |= EXT4_MB_DELALLOC_RESERVED; 450360e58e0fSMingming Cao else { 450460e58e0fSMingming Cao /* Without delayed allocation we need to verify 450560e58e0fSMingming Cao * there is enough free blocks to do block allocation 450660e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 4507d2a17637SMingming Cao */ 4508030ba6bcSAneesh Kumar K.V while (ar->len && ext4_claim_free_blocks(sbi, ar->len)) { 4509030ba6bcSAneesh Kumar K.V /* let others to free the space */ 4510030ba6bcSAneesh Kumar K.V yield(); 4511030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 4512030ba6bcSAneesh Kumar K.V } 4513030ba6bcSAneesh Kumar K.V if (!ar->len) { 451407031431SMingming Cao *errp = -ENOSPC; 451507031431SMingming Cao return 0; 451607031431SMingming Cao } 45176bc6e63fSAneesh Kumar K.V reserv_blks = ar->len; 4518a269eb18SJan Kara while (ar->len && vfs_dq_alloc_block(ar->inode, ar->len)) { 4519c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4520c9de560dSAlex Tomas ar->len--; 4521c9de560dSAlex Tomas } 452260e58e0fSMingming Cao inquota = ar->len; 4523c9de560dSAlex Tomas if (ar->len == 0) { 4524c9de560dSAlex Tomas *errp = -EDQUOT; 45250087d9fbSAneesh Kumar K.V goto out3; 4526c9de560dSAlex Tomas } 452760e58e0fSMingming Cao } 4528d2a17637SMingming Cao 4529256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 4530833576b3STheodore Ts'o if (!ac) { 4531363d4251SShen Feng ar->len = 0; 4532256bdb49SEric Sandeen *errp = -ENOMEM; 4533363d4251SShen Feng goto out1; 4534256bdb49SEric Sandeen } 4535256bdb49SEric Sandeen 4536256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4537c9de560dSAlex Tomas if (*errp) { 4538c9de560dSAlex Tomas ar->len = 0; 4539363d4251SShen Feng goto out2; 4540c9de560dSAlex Tomas } 4541c9de560dSAlex Tomas 4542256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 4543256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4544256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4545256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 4546c9de560dSAlex Tomas repeat: 4547c9de560dSAlex Tomas /* allocate space in core */ 4548256bdb49SEric Sandeen ext4_mb_regular_allocator(ac); 4549c9de560dSAlex Tomas 4550c9de560dSAlex Tomas /* as we've just preallocated more space than 4551c9de560dSAlex Tomas * user requested orinally, we store allocated 4552c9de560dSAlex Tomas * space in a special descriptor */ 4553256bdb49SEric Sandeen if (ac->ac_status == AC_STATUS_FOUND && 4554256bdb49SEric Sandeen ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 4555256bdb49SEric Sandeen ext4_mb_new_preallocation(ac); 4556c9de560dSAlex Tomas } 4557256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 45586bc6e63fSAneesh Kumar K.V *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_blks); 4559519deca0SAneesh Kumar K.V if (*errp == -EAGAIN) { 45608556e8f3SAneesh Kumar K.V /* 45618556e8f3SAneesh Kumar K.V * drop the reference that we took 45628556e8f3SAneesh Kumar K.V * in ext4_mb_use_best_found 45638556e8f3SAneesh Kumar K.V */ 45648556e8f3SAneesh Kumar K.V ext4_mb_release_context(ac); 4565519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_group = 0; 4566519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_start = 0; 4567519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len = 0; 4568519deca0SAneesh Kumar K.V ac->ac_status = AC_STATUS_CONTINUE; 4569519deca0SAneesh Kumar K.V goto repeat; 4570519deca0SAneesh Kumar K.V } else if (*errp) { 4571519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len = 0; 4572519deca0SAneesh Kumar K.V ar->len = 0; 4573519deca0SAneesh Kumar K.V ext4_mb_show_ac(ac); 4574519deca0SAneesh Kumar K.V } else { 4575256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4576256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4577519deca0SAneesh Kumar K.V } 4578c9de560dSAlex Tomas } else { 4579256bdb49SEric Sandeen freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 4580c9de560dSAlex Tomas if (freed) 4581c9de560dSAlex Tomas goto repeat; 4582c9de560dSAlex Tomas *errp = -ENOSPC; 4583256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4584c9de560dSAlex Tomas ar->len = 0; 4585256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4586c9de560dSAlex Tomas } 4587c9de560dSAlex Tomas 4588256bdb49SEric Sandeen ext4_mb_release_context(ac); 4589c9de560dSAlex Tomas 4590363d4251SShen Feng out2: 4591363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 4592363d4251SShen Feng out1: 459360e58e0fSMingming Cao if (inquota && ar->len < inquota) 4594a269eb18SJan Kara vfs_dq_free_block(ar->inode, inquota - ar->len); 45950087d9fbSAneesh Kumar K.V out3: 45960087d9fbSAneesh Kumar K.V if (!ar->len) { 45970087d9fbSAneesh Kumar K.V if (!EXT4_I(ar->inode)->i_delalloc_reserved_flag) 45980087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 45990087d9fbSAneesh Kumar K.V percpu_counter_sub(&sbi->s_dirtyblocks_counter, 46000087d9fbSAneesh Kumar K.V reserv_blks); 46010087d9fbSAneesh Kumar K.V } 4602c9de560dSAlex Tomas 46039bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 4604ba80b101STheodore Ts'o 4605c9de560dSAlex Tomas return block; 4606c9de560dSAlex Tomas } 4607c9de560dSAlex Tomas 4608c894058dSAneesh Kumar K.V /* 4609c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 4610c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 4611c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 4612c894058dSAneesh Kumar K.V */ 4613c894058dSAneesh Kumar K.V static int can_merge(struct ext4_free_data *entry1, 4614c894058dSAneesh Kumar K.V struct ext4_free_data *entry2) 4615c894058dSAneesh Kumar K.V { 4616c894058dSAneesh Kumar K.V if ((entry1->t_tid == entry2->t_tid) && 4617c894058dSAneesh Kumar K.V (entry1->group == entry2->group) && 4618c894058dSAneesh Kumar K.V ((entry1->start_blk + entry1->count) == entry2->start_blk)) 4619c894058dSAneesh Kumar K.V return 1; 4620c894058dSAneesh Kumar K.V return 0; 4621c894058dSAneesh Kumar K.V } 4622c894058dSAneesh Kumar K.V 46234ddfef7bSEric Sandeen static noinline_for_stack int 46244ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 46257a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 4626c9de560dSAlex Tomas { 46277a2fcbf7SAneesh Kumar K.V ext4_grpblk_t block; 46287a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 4629c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4630c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4631c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4632c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 4633c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 4634c894058dSAneesh Kumar K.V 46350390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 4636c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4637c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4638c9de560dSAlex Tomas 4639c894058dSAneesh Kumar K.V new_node = &new_entry->node; 46407a2fcbf7SAneesh Kumar K.V block = new_entry->start_blk; 4641c9de560dSAlex Tomas 4642c894058dSAneesh Kumar K.V if (!*n) { 4643c894058dSAneesh Kumar K.V /* first free block exent. We need to 4644c894058dSAneesh Kumar K.V protect buddy cache from being freed, 4645c9de560dSAlex Tomas * otherwise we'll refresh it from 4646c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4647c9de560dSAlex Tomas * blocks */ 4648c9de560dSAlex Tomas page_cache_get(e4b->bd_buddy_page); 4649c9de560dSAlex Tomas page_cache_get(e4b->bd_bitmap_page); 4650c894058dSAneesh Kumar K.V } 4651c894058dSAneesh Kumar K.V while (*n) { 4652c894058dSAneesh Kumar K.V parent = *n; 4653c894058dSAneesh Kumar K.V entry = rb_entry(parent, struct ext4_free_data, node); 4654c894058dSAneesh Kumar K.V if (block < entry->start_blk) 4655c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 4656c894058dSAneesh Kumar K.V else if (block >= (entry->start_blk + entry->count)) 4657c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 4658c894058dSAneesh Kumar K.V else { 46595d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, __func__, 4660fde4d95aSTheodore Ts'o "Double free of blocks %d (%d %d)", 4661c894058dSAneesh Kumar K.V block, entry->start_blk, entry->count); 4662c894058dSAneesh Kumar K.V return 0; 4663c9de560dSAlex Tomas } 4664c9de560dSAlex Tomas } 4665c9de560dSAlex Tomas 4666c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 4667c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 4668c894058dSAneesh Kumar K.V 4669c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 4670c894058dSAneesh Kumar K.V node = rb_prev(new_node); 4671c894058dSAneesh Kumar K.V if (node) { 4672c894058dSAneesh Kumar K.V entry = rb_entry(node, struct ext4_free_data, node); 4673c894058dSAneesh Kumar K.V if (can_merge(entry, new_entry)) { 4674c894058dSAneesh Kumar K.V new_entry->start_blk = entry->start_blk; 4675c894058dSAneesh Kumar K.V new_entry->count += entry->count; 4676c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 4677c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 4678c894058dSAneesh Kumar K.V list_del(&entry->list); 4679c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4680c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 4681c9de560dSAlex Tomas } 4682c9de560dSAlex Tomas } 4683c894058dSAneesh Kumar K.V 4684c894058dSAneesh Kumar K.V node = rb_next(new_node); 4685c894058dSAneesh Kumar K.V if (node) { 4686c894058dSAneesh Kumar K.V entry = rb_entry(node, struct ext4_free_data, node); 4687c894058dSAneesh Kumar K.V if (can_merge(new_entry, entry)) { 4688c894058dSAneesh Kumar K.V new_entry->count += entry->count; 4689c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 4690c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 4691c894058dSAneesh Kumar K.V list_del(&entry->list); 4692c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4693c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 4694c894058dSAneesh Kumar K.V } 4695c894058dSAneesh Kumar K.V } 46963e624fc7STheodore Ts'o /* Add the extent to transaction's private list */ 4697c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 46983e624fc7STheodore Ts'o list_add(&new_entry->list, &handle->h_transaction->t_private_list); 4699c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4700c9de560dSAlex Tomas return 0; 4701c9de560dSAlex Tomas } 4702c9de560dSAlex Tomas 4703c9de560dSAlex Tomas /* 4704c9de560dSAlex Tomas * Main entry point into mballoc to free blocks 4705c9de560dSAlex Tomas */ 4706c9de560dSAlex Tomas void ext4_mb_free_blocks(handle_t *handle, struct inode *inode, 47070610b6e9STheodore Ts'o ext4_fsblk_t block, unsigned long count, 4708c9de560dSAlex Tomas int metadata, unsigned long *freed) 4709c9de560dSAlex Tomas { 471026346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 4711c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4712256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4713c9de560dSAlex Tomas struct ext4_group_desc *gdp; 4714c9de560dSAlex Tomas struct ext4_super_block *es; 4715498e5f24STheodore Ts'o unsigned int overflow; 4716c9de560dSAlex Tomas ext4_grpblk_t bit; 4717c9de560dSAlex Tomas struct buffer_head *gd_bh; 4718c9de560dSAlex Tomas ext4_group_t block_group; 4719c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4720c9de560dSAlex Tomas struct ext4_buddy e4b; 4721c9de560dSAlex Tomas int err = 0; 4722c9de560dSAlex Tomas int ret; 4723c9de560dSAlex Tomas 4724c9de560dSAlex Tomas *freed = 0; 4725c9de560dSAlex Tomas 4726c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4727c9de560dSAlex Tomas es = EXT4_SB(sb)->s_es; 4728c9de560dSAlex Tomas if (block < le32_to_cpu(es->s_first_data_block) || 4729c9de560dSAlex Tomas block + count < block || 4730c9de560dSAlex Tomas block + count > ext4_blocks_count(es)) { 473146e665e9SHarvey Harrison ext4_error(sb, __func__, 4732c9de560dSAlex Tomas "Freeing blocks not in datazone - " 47330610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 4734c9de560dSAlex Tomas goto error_return; 4735c9de560dSAlex Tomas } 4736c9de560dSAlex Tomas 47370610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 47389bffad1eSTheodore Ts'o trace_ext4_free_blocks(inode, block, count, metadata); 4739c9de560dSAlex Tomas 4740256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 4741256bdb49SEric Sandeen if (ac) { 4742256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_FREE; 4743256bdb49SEric Sandeen ac->ac_inode = inode; 4744256bdb49SEric Sandeen ac->ac_sb = sb; 4745256bdb49SEric Sandeen } 4746c9de560dSAlex Tomas 4747c9de560dSAlex Tomas do_more: 4748c9de560dSAlex Tomas overflow = 0; 4749c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 4750c9de560dSAlex Tomas 4751c9de560dSAlex Tomas /* 4752c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 4753c9de560dSAlex Tomas * boundary. 4754c9de560dSAlex Tomas */ 4755c9de560dSAlex Tomas if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) { 4756c9de560dSAlex Tomas overflow = bit + count - EXT4_BLOCKS_PER_GROUP(sb); 4757c9de560dSAlex Tomas count -= overflow; 4758c9de560dSAlex Tomas } 4759574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 4760ce89f46cSAneesh Kumar K.V if (!bitmap_bh) { 4761ce89f46cSAneesh Kumar K.V err = -EIO; 4762c9de560dSAlex Tomas goto error_return; 4763ce89f46cSAneesh Kumar K.V } 4764c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 4765ce89f46cSAneesh Kumar K.V if (!gdp) { 4766ce89f46cSAneesh Kumar K.V err = -EIO; 4767c9de560dSAlex Tomas goto error_return; 4768ce89f46cSAneesh Kumar K.V } 4769c9de560dSAlex Tomas 4770c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 4771c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 4772c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 4773c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group) || 4774c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 4775c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group)) { 4776c9de560dSAlex Tomas 477746e665e9SHarvey Harrison ext4_error(sb, __func__, 4778c9de560dSAlex Tomas "Freeing blocks in system zone - " 47790610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 4780519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 4781519deca0SAneesh Kumar K.V goto error_return; 4782c9de560dSAlex Tomas } 4783c9de560dSAlex Tomas 4784c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 4785c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 4786c9de560dSAlex Tomas if (err) 4787c9de560dSAlex Tomas goto error_return; 4788c9de560dSAlex Tomas 4789c9de560dSAlex Tomas /* 4790c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 4791c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 4792c9de560dSAlex Tomas * using it 4793c9de560dSAlex Tomas */ 4794c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 4795c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 4796c9de560dSAlex Tomas if (err) 4797c9de560dSAlex Tomas goto error_return; 4798c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4799c9de560dSAlex Tomas { 4800c9de560dSAlex Tomas int i; 4801c9de560dSAlex Tomas for (i = 0; i < count; i++) 4802c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 4803c9de560dSAlex Tomas } 4804c9de560dSAlex Tomas #endif 4805256bdb49SEric Sandeen if (ac) { 4806256bdb49SEric Sandeen ac->ac_b_ex.fe_group = block_group; 4807256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 4808256bdb49SEric Sandeen ac->ac_b_ex.fe_len = count; 4809256bdb49SEric Sandeen ext4_mb_store_history(ac); 4810256bdb49SEric Sandeen } 4811c9de560dSAlex Tomas 4812920313a7SAneesh Kumar K.V err = ext4_mb_load_buddy(sb, block_group, &e4b); 4813920313a7SAneesh Kumar K.V if (err) 4814920313a7SAneesh Kumar K.V goto error_return; 48150390131bSFrank Mayhar if (metadata && ext4_handle_valid(handle)) { 48167a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 48177a2fcbf7SAneesh Kumar K.V /* 48187a2fcbf7SAneesh Kumar K.V * blocks being freed are metadata. these blocks shouldn't 48197a2fcbf7SAneesh Kumar K.V * be used until this transaction is committed 48207a2fcbf7SAneesh Kumar K.V */ 48217a2fcbf7SAneesh Kumar K.V new_entry = kmem_cache_alloc(ext4_free_ext_cachep, GFP_NOFS); 48227a2fcbf7SAneesh Kumar K.V new_entry->start_blk = bit; 48237a2fcbf7SAneesh Kumar K.V new_entry->group = block_group; 48247a2fcbf7SAneesh Kumar K.V new_entry->count = count; 48257a2fcbf7SAneesh Kumar K.V new_entry->t_tid = handle->h_transaction->t_tid; 4826955ce5f5SAneesh Kumar K.V 48277a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 4828955ce5f5SAneesh Kumar K.V mb_clear_bits(bitmap_bh->b_data, bit, count); 48297a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 4830c9de560dSAlex Tomas } else { 48317a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 48327a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 48337a2fcbf7SAneesh Kumar K.V * them with group lock_held 48347a2fcbf7SAneesh Kumar K.V */ 4835955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 4836955ce5f5SAneesh Kumar K.V mb_clear_bits(bitmap_bh->b_data, bit, count); 48377e5a8cddSShen Feng mb_free_blocks(inode, &e4b, bit, count); 4838c9de560dSAlex Tomas ext4_mb_return_to_preallocation(inode, &e4b, block, count); 4839c9de560dSAlex Tomas } 4840c9de560dSAlex Tomas 4841560671a0SAneesh Kumar K.V ret = ext4_free_blks_count(sb, gdp) + count; 4842560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, ret); 4843c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp); 4844955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 4845c9de560dSAlex Tomas percpu_counter_add(&sbi->s_freeblocks_counter, count); 4846c9de560dSAlex Tomas 4847772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 4848772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 48499f24e420STheodore Ts'o atomic_add(count, &sbi->s_flex_groups[flex_group].free_blocks); 4850772cb7c8SJose R. Santos } 4851772cb7c8SJose R. Santos 4852c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4853c9de560dSAlex Tomas 4854c9de560dSAlex Tomas *freed += count; 4855c9de560dSAlex Tomas 48567a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 48577a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 48587a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 48597a2fcbf7SAneesh Kumar K.V 4860c9de560dSAlex Tomas /* And the group descriptor block */ 4861c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 48620390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 4863c9de560dSAlex Tomas if (!err) 4864c9de560dSAlex Tomas err = ret; 4865c9de560dSAlex Tomas 4866c9de560dSAlex Tomas if (overflow && !err) { 4867c9de560dSAlex Tomas block += count; 4868c9de560dSAlex Tomas count = overflow; 4869c9de560dSAlex Tomas put_bh(bitmap_bh); 4870c9de560dSAlex Tomas goto do_more; 4871c9de560dSAlex Tomas } 4872c9de560dSAlex Tomas sb->s_dirt = 1; 4873c9de560dSAlex Tomas error_return: 4874c9de560dSAlex Tomas brelse(bitmap_bh); 4875c9de560dSAlex Tomas ext4_std_error(sb, err); 4876256bdb49SEric Sandeen if (ac) 4877256bdb49SEric Sandeen kmem_cache_free(ext4_ac_cachep, ac); 4878c9de560dSAlex Tomas return; 4879c9de560dSAlex Tomas } 4880