1c9de560dSAlex Tomas /* 2c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 3c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 4c9de560dSAlex Tomas * 5c9de560dSAlex Tomas * This program is free software; you can redistribute it and/or modify 6c9de560dSAlex Tomas * it under the terms of the GNU General Public License version 2 as 7c9de560dSAlex Tomas * published by the Free Software Foundation. 8c9de560dSAlex Tomas * 9c9de560dSAlex Tomas * This program is distributed in the hope that it will be useful, 10c9de560dSAlex Tomas * but WITHOUT ANY WARRANTY; without even the implied warranty of 11c9de560dSAlex Tomas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12c9de560dSAlex Tomas * GNU General Public License for more details. 13c9de560dSAlex Tomas * 14c9de560dSAlex Tomas * You should have received a copy of the GNU General Public Licens 15c9de560dSAlex Tomas * along with this program; if not, write to the Free Software 16c9de560dSAlex Tomas * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- 17c9de560dSAlex Tomas */ 18c9de560dSAlex Tomas 19c9de560dSAlex Tomas 20c9de560dSAlex Tomas /* 21c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 22c9de560dSAlex Tomas */ 23c9de560dSAlex Tomas 24c9de560dSAlex Tomas #include <linux/time.h> 25c9de560dSAlex Tomas #include <linux/fs.h> 26c9de560dSAlex Tomas #include <linux/namei.h> 27c9de560dSAlex Tomas #include <linux/ext4_jbd2.h> 28c9de560dSAlex Tomas #include <linux/ext4_fs.h> 29c9de560dSAlex Tomas #include <linux/quotaops.h> 30c9de560dSAlex Tomas #include <linux/buffer_head.h> 31c9de560dSAlex Tomas #include <linux/module.h> 32c9de560dSAlex Tomas #include <linux/swap.h> 33c9de560dSAlex Tomas #include <linux/proc_fs.h> 34c9de560dSAlex Tomas #include <linux/pagemap.h> 35c9de560dSAlex Tomas #include <linux/seq_file.h> 36c9de560dSAlex Tomas #include <linux/version.h> 37c9de560dSAlex Tomas #include "group.h" 38c9de560dSAlex Tomas 39c9de560dSAlex Tomas /* 40c9de560dSAlex Tomas * MUSTDO: 41c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 42c9de560dSAlex Tomas * - search for metadata in few groups 43c9de560dSAlex Tomas * 44c9de560dSAlex Tomas * TODO v4: 45c9de560dSAlex Tomas * - normalization should take into account whether file is still open 46c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 47c9de560dSAlex Tomas * - don't normalize tails 48c9de560dSAlex Tomas * - quota 49c9de560dSAlex Tomas * - reservation for superuser 50c9de560dSAlex Tomas * 51c9de560dSAlex Tomas * TODO v3: 52c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 53c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 54c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 55c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 56c9de560dSAlex Tomas * - error handling 57c9de560dSAlex Tomas */ 58c9de560dSAlex Tomas 59c9de560dSAlex Tomas /* 60c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 61c9de560dSAlex Tomas * near to the goal(block) value specified. 62c9de560dSAlex Tomas * 63c9de560dSAlex Tomas * During initialization phase of the allocator we decide to use the group 64c9de560dSAlex Tomas * preallocation or inode preallocation depending on the size file. The 65c9de560dSAlex Tomas * size of the file could be the resulting file size we would have after 66c9de560dSAlex Tomas * allocation or the current file size which ever is larger. If the size is 67c9de560dSAlex Tomas * less that sbi->s_mb_stream_request we select the group 68c9de560dSAlex Tomas * preallocation. The default value of s_mb_stream_request is 16 69c9de560dSAlex Tomas * blocks. This can also be tuned via 70c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/stream_req. The value is represented in terms 71c9de560dSAlex Tomas * of number of blocks. 72c9de560dSAlex Tomas * 73c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 74c9de560dSAlex Tomas * ensure that we have small file closer in the disk. 75c9de560dSAlex Tomas * 76c9de560dSAlex Tomas * First stage the allocator looks at the inode prealloc list 77c9de560dSAlex Tomas * ext4_inode_info->i_prealloc_list contain list of prealloc spaces for 78c9de560dSAlex Tomas * this particular inode. The inode prealloc space is represented as: 79c9de560dSAlex Tomas * 80c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 81c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 82c9de560dSAlex Tomas * pa_len -> lenght for this prealloc space 83c9de560dSAlex Tomas * pa_free -> free space available in this prealloc space 84c9de560dSAlex Tomas * 85c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 86c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 87c9de560dSAlex Tomas * space we will consume the particular prealloc space. This make sure that 88c9de560dSAlex Tomas * that the we have contiguous physical blocks representing the file blocks 89c9de560dSAlex Tomas * 90c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 91c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 92c9de560dSAlex Tomas * pa_free. 93c9de560dSAlex Tomas * 94c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 95c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 96c9de560dSAlex Tomas * prealloc space. These are per CPU prealloc list repreasented as 97c9de560dSAlex Tomas * 98c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 99c9de560dSAlex Tomas * 100c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 101c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 102c9de560dSAlex Tomas * 103c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 104c9de560dSAlex Tomas * enough free space (pa_free) withing the prealloc space. 105c9de560dSAlex Tomas * 106c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 107c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 108c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 109c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 110c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 111c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 112c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 113c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 114c9de560dSAlex Tomas * inode as: 115c9de560dSAlex Tomas * 116c9de560dSAlex Tomas * { page } 117c9de560dSAlex Tomas * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]... 118c9de560dSAlex Tomas * 119c9de560dSAlex Tomas * 120c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 121c9de560dSAlex Tomas * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE / 122c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 123c9de560dSAlex Tomas * which is blocks_per_page/2 124c9de560dSAlex Tomas * 125c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 126c9de560dSAlex Tomas * away when the filesystem is unmounted. 127c9de560dSAlex Tomas * 128c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 129c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 130c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 131c9de560dSAlex Tomas * 132c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 133c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 134c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 135c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 136c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 137c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 138c9de560dSAlex Tomas * sbi->s_mb_group_prealloc. Default value of s_mb_group_prealloc is set to 139c9de560dSAlex Tomas * 512 blocks. This can be tuned via 140c9de560dSAlex Tomas * /proc/fs/ext4/<partition/group_prealloc. The value is represented in 141c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 142c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 143c9de560dSAlex Tomas * stripe value (sbi->s_stripe) 144c9de560dSAlex Tomas * 145c9de560dSAlex Tomas * The regular allocator(using the buddy cache) support few tunables. 146c9de560dSAlex Tomas * 147c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/min_to_scan 148c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/max_to_scan 149c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/order2_req 150c9de560dSAlex Tomas * 151c9de560dSAlex Tomas * The regular allocator use buddy scan only if the request len is power of 152c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 153c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 154c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/order2_req. If the request len is equal to 155c9de560dSAlex Tomas * stripe size (sbi->s_stripe), we try to search for contigous block in 156c9de560dSAlex Tomas * stripe size. This should result in better allocation on RAID setup. If 157c9de560dSAlex Tomas * not we search in the specific group using bitmap for best extents. The 158c9de560dSAlex Tomas * tunable min_to_scan and max_to_scan controll the behaviour here. 159c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 160c9de560dSAlex Tomas * extent and max_to_scanindicate how long the mballoc __can__ look for a 161c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 162c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 163c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 164c9de560dSAlex Tomas * can used for allocation. ext4_mb_good_group explains how the groups are 165c9de560dSAlex Tomas * checked. 166c9de560dSAlex Tomas * 167c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 168c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 169c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 170c9de560dSAlex Tomas * subsequent request. 171c9de560dSAlex Tomas */ 172c9de560dSAlex Tomas 173c9de560dSAlex Tomas /* 174c9de560dSAlex Tomas * mballoc operates on the following data: 175c9de560dSAlex Tomas * - on-disk bitmap 176c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 177c9de560dSAlex Tomas * - preallocation descriptors (PAs) 178c9de560dSAlex Tomas * 179c9de560dSAlex Tomas * there are two types of preallocations: 180c9de560dSAlex Tomas * - inode 181c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 182c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 183c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 184c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 185c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 186c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 187c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 188c9de560dSAlex Tomas * must discard all preallocated blocks. 189c9de560dSAlex Tomas * - locality group 190c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 191c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 192c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 193c9de560dSAlex Tomas * it's consumed from the beginning to the end. 194c9de560dSAlex Tomas * 195c9de560dSAlex Tomas * relation between them can be expressed as: 196c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 197c9de560dSAlex Tomas * 198c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 199c9de560dSAlex Tomas * - allocated blocks (persistent) 200c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 201c9de560dSAlex Tomas * 202c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 203c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 204c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 205c9de560dSAlex Tomas * 206c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 207c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 208c9de560dSAlex Tomas * 209c9de560dSAlex Tomas * all operations can be expressed as: 210c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 211c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 212c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 213c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 214c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 215c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 216c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 217c9de560dSAlex Tomas * is used in real operation because we can't know actual used 218c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 219c9de560dSAlex Tomas * 220c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 221c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 222c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 223c9de560dSAlex Tomas * the following knowledge: 224c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 225c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 226c9de560dSAlex Tomas * nobody can re-allocate that block 227c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 228c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 229c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 230c9de560dSAlex Tomas * block 231c9de560dSAlex Tomas * 232c9de560dSAlex Tomas * so, now we're building a concurrency table: 233c9de560dSAlex Tomas * - init buddy vs. 234c9de560dSAlex Tomas * - new PA 235c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 236c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 237c9de560dSAlex Tomas * - use inode PA 238c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 239c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 240c9de560dSAlex Tomas * - discard inode PA 241c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 242c9de560dSAlex Tomas * - use locality group PA 243c9de560dSAlex Tomas * again PA-=N must be serialized with init 244c9de560dSAlex Tomas * - discard locality group PA 245c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 246c9de560dSAlex Tomas * - new PA vs. 247c9de560dSAlex Tomas * - use inode PA 248c9de560dSAlex Tomas * i_data_sem serializes them 249c9de560dSAlex Tomas * - discard inode PA 250c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 251c9de560dSAlex Tomas * - use locality group PA 252c9de560dSAlex Tomas * some mutex should serialize them 253c9de560dSAlex Tomas * - discard locality group PA 254c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 255c9de560dSAlex Tomas * - use inode PA 256c9de560dSAlex Tomas * - use inode PA 257c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 258c9de560dSAlex Tomas * - discard inode PA 259c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 260c9de560dSAlex Tomas * - use locality group PA 261c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 262c9de560dSAlex Tomas * - discard locality group PA 263c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 264c9de560dSAlex Tomas * 265c9de560dSAlex Tomas * now we're ready to make few consequences: 266c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 267c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 268c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 269c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 270c9de560dSAlex Tomas * any discard or they're serialized somehow 271c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 272c9de560dSAlex Tomas * 273c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 274c9de560dSAlex Tomas * in this case, but we should care about concurrent init 275c9de560dSAlex Tomas * 276c9de560dSAlex Tomas */ 277c9de560dSAlex Tomas 278c9de560dSAlex Tomas /* 279c9de560dSAlex Tomas * Logic in few words: 280c9de560dSAlex Tomas * 281c9de560dSAlex Tomas * - allocation: 282c9de560dSAlex Tomas * load group 283c9de560dSAlex Tomas * find blocks 284c9de560dSAlex Tomas * mark bits in on-disk bitmap 285c9de560dSAlex Tomas * release group 286c9de560dSAlex Tomas * 287c9de560dSAlex Tomas * - use preallocation: 288c9de560dSAlex Tomas * find proper PA (per-inode or group) 289c9de560dSAlex Tomas * load group 290c9de560dSAlex Tomas * mark bits in on-disk bitmap 291c9de560dSAlex Tomas * release group 292c9de560dSAlex Tomas * release PA 293c9de560dSAlex Tomas * 294c9de560dSAlex Tomas * - free: 295c9de560dSAlex Tomas * load group 296c9de560dSAlex Tomas * mark bits in on-disk bitmap 297c9de560dSAlex Tomas * release group 298c9de560dSAlex Tomas * 299c9de560dSAlex Tomas * - discard preallocations in group: 300c9de560dSAlex Tomas * mark PAs deleted 301c9de560dSAlex Tomas * move them onto local list 302c9de560dSAlex Tomas * load on-disk bitmap 303c9de560dSAlex Tomas * load group 304c9de560dSAlex Tomas * remove PA from object (inode or locality group) 305c9de560dSAlex Tomas * mark free blocks in-core 306c9de560dSAlex Tomas * 307c9de560dSAlex Tomas * - discard inode's preallocations: 308c9de560dSAlex Tomas */ 309c9de560dSAlex Tomas 310c9de560dSAlex Tomas /* 311c9de560dSAlex Tomas * Locking rules 312c9de560dSAlex Tomas * 313c9de560dSAlex Tomas * Locks: 314c9de560dSAlex Tomas * - bitlock on a group (group) 315c9de560dSAlex Tomas * - object (inode/locality) (object) 316c9de560dSAlex Tomas * - per-pa lock (pa) 317c9de560dSAlex Tomas * 318c9de560dSAlex Tomas * Paths: 319c9de560dSAlex Tomas * - new pa 320c9de560dSAlex Tomas * object 321c9de560dSAlex Tomas * group 322c9de560dSAlex Tomas * 323c9de560dSAlex Tomas * - find and use pa: 324c9de560dSAlex Tomas * pa 325c9de560dSAlex Tomas * 326c9de560dSAlex Tomas * - release consumed pa: 327c9de560dSAlex Tomas * pa 328c9de560dSAlex Tomas * group 329c9de560dSAlex Tomas * object 330c9de560dSAlex Tomas * 331c9de560dSAlex Tomas * - generate in-core bitmap: 332c9de560dSAlex Tomas * group 333c9de560dSAlex Tomas * pa 334c9de560dSAlex Tomas * 335c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 336c9de560dSAlex Tomas * object 337c9de560dSAlex Tomas * pa 338c9de560dSAlex Tomas * group 339c9de560dSAlex Tomas * 340c9de560dSAlex Tomas * - discard all for given group: 341c9de560dSAlex Tomas * group 342c9de560dSAlex Tomas * pa 343c9de560dSAlex Tomas * group 344c9de560dSAlex Tomas * object 345c9de560dSAlex Tomas * 346c9de560dSAlex Tomas */ 347c9de560dSAlex Tomas 348c9de560dSAlex Tomas /* 349c9de560dSAlex Tomas * with AGGRESSIVE_CHECK allocator runs consistency checks over 350c9de560dSAlex Tomas * structures. these checks slow things down a lot 351c9de560dSAlex Tomas */ 352c9de560dSAlex Tomas #define AGGRESSIVE_CHECK__ 353c9de560dSAlex Tomas 354c9de560dSAlex Tomas /* 355c9de560dSAlex Tomas * with DOUBLE_CHECK defined mballoc creates persistent in-core 356c9de560dSAlex Tomas * bitmaps, maintains and uses them to check for double allocations 357c9de560dSAlex Tomas */ 358c9de560dSAlex Tomas #define DOUBLE_CHECK__ 359c9de560dSAlex Tomas 360c9de560dSAlex Tomas /* 361c9de560dSAlex Tomas */ 362c9de560dSAlex Tomas #define MB_DEBUG__ 363c9de560dSAlex Tomas #ifdef MB_DEBUG 364c9de560dSAlex Tomas #define mb_debug(fmt, a...) printk(fmt, ##a) 365c9de560dSAlex Tomas #else 366c9de560dSAlex Tomas #define mb_debug(fmt, a...) 367c9de560dSAlex Tomas #endif 368c9de560dSAlex Tomas 369c9de560dSAlex Tomas /* 370c9de560dSAlex Tomas * with EXT4_MB_HISTORY mballoc stores last N allocations in memory 371c9de560dSAlex Tomas * and you can monitor it in /proc/fs/ext4/<dev>/mb_history 372c9de560dSAlex Tomas */ 373c9de560dSAlex Tomas #define EXT4_MB_HISTORY 374c9de560dSAlex Tomas #define EXT4_MB_HISTORY_ALLOC 1 /* allocation */ 375c9de560dSAlex Tomas #define EXT4_MB_HISTORY_PREALLOC 2 /* preallocated blocks used */ 376c9de560dSAlex Tomas #define EXT4_MB_HISTORY_DISCARD 4 /* preallocation discarded */ 377c9de560dSAlex Tomas #define EXT4_MB_HISTORY_FREE 8 /* free */ 378c9de560dSAlex Tomas 379c9de560dSAlex Tomas #define EXT4_MB_HISTORY_DEFAULT (EXT4_MB_HISTORY_ALLOC | \ 380c9de560dSAlex Tomas EXT4_MB_HISTORY_PREALLOC) 381c9de560dSAlex Tomas 382c9de560dSAlex Tomas /* 383c9de560dSAlex Tomas * How long mballoc can look for a best extent (in found extents) 384c9de560dSAlex Tomas */ 385c9de560dSAlex Tomas #define MB_DEFAULT_MAX_TO_SCAN 200 386c9de560dSAlex Tomas 387c9de560dSAlex Tomas /* 388c9de560dSAlex Tomas * How long mballoc must look for a best extent 389c9de560dSAlex Tomas */ 390c9de560dSAlex Tomas #define MB_DEFAULT_MIN_TO_SCAN 10 391c9de560dSAlex Tomas 392c9de560dSAlex Tomas /* 393c9de560dSAlex Tomas * How many groups mballoc will scan looking for the best chunk 394c9de560dSAlex Tomas */ 395c9de560dSAlex Tomas #define MB_DEFAULT_MAX_GROUPS_TO_SCAN 5 396c9de560dSAlex Tomas 397c9de560dSAlex Tomas /* 398c9de560dSAlex Tomas * with 'ext4_mb_stats' allocator will collect stats that will be 399c9de560dSAlex Tomas * shown at umount. The collecting costs though! 400c9de560dSAlex Tomas */ 401c9de560dSAlex Tomas #define MB_DEFAULT_STATS 1 402c9de560dSAlex Tomas 403c9de560dSAlex Tomas /* 404c9de560dSAlex Tomas * files smaller than MB_DEFAULT_STREAM_THRESHOLD are served 405c9de560dSAlex Tomas * by the stream allocator, which purpose is to pack requests 406c9de560dSAlex Tomas * as close each to other as possible to produce smooth I/O traffic 407c9de560dSAlex Tomas * We use locality group prealloc space for stream request. 408c9de560dSAlex Tomas * We can tune the same via /proc/fs/ext4/<parition>/stream_req 409c9de560dSAlex Tomas */ 410c9de560dSAlex Tomas #define MB_DEFAULT_STREAM_THRESHOLD 16 /* 64K */ 411c9de560dSAlex Tomas 412c9de560dSAlex Tomas /* 413c9de560dSAlex Tomas * for which requests use 2^N search using buddies 414c9de560dSAlex Tomas */ 415c9de560dSAlex Tomas #define MB_DEFAULT_ORDER2_REQS 2 416c9de560dSAlex Tomas 417c9de560dSAlex Tomas /* 418c9de560dSAlex Tomas * default group prealloc size 512 blocks 419c9de560dSAlex Tomas */ 420c9de560dSAlex Tomas #define MB_DEFAULT_GROUP_PREALLOC 512 421c9de560dSAlex Tomas 422c9de560dSAlex Tomas static struct kmem_cache *ext4_pspace_cachep; 423256bdb49SEric Sandeen static struct kmem_cache *ext4_ac_cachep; 424c9de560dSAlex Tomas 425c9de560dSAlex Tomas #ifdef EXT4_BB_MAX_BLOCKS 426c9de560dSAlex Tomas #undef EXT4_BB_MAX_BLOCKS 427c9de560dSAlex Tomas #endif 428c9de560dSAlex Tomas #define EXT4_BB_MAX_BLOCKS 30 429c9de560dSAlex Tomas 430c9de560dSAlex Tomas struct ext4_free_metadata { 431c9de560dSAlex Tomas ext4_group_t group; 432c9de560dSAlex Tomas unsigned short num; 433c9de560dSAlex Tomas ext4_grpblk_t blocks[EXT4_BB_MAX_BLOCKS]; 434c9de560dSAlex Tomas struct list_head list; 435c9de560dSAlex Tomas }; 436c9de560dSAlex Tomas 437c9de560dSAlex Tomas struct ext4_group_info { 438c9de560dSAlex Tomas unsigned long bb_state; 439c9de560dSAlex Tomas unsigned long bb_tid; 440c9de560dSAlex Tomas struct ext4_free_metadata *bb_md_cur; 441c9de560dSAlex Tomas unsigned short bb_first_free; 442c9de560dSAlex Tomas unsigned short bb_free; 443c9de560dSAlex Tomas unsigned short bb_fragments; 444c9de560dSAlex Tomas struct list_head bb_prealloc_list; 445c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 446c9de560dSAlex Tomas void *bb_bitmap; 447c9de560dSAlex Tomas #endif 448c9de560dSAlex Tomas unsigned short bb_counters[]; 449c9de560dSAlex Tomas }; 450c9de560dSAlex Tomas 451c9de560dSAlex Tomas #define EXT4_GROUP_INFO_NEED_INIT_BIT 0 452c9de560dSAlex Tomas #define EXT4_GROUP_INFO_LOCKED_BIT 1 453c9de560dSAlex Tomas 454c9de560dSAlex Tomas #define EXT4_MB_GRP_NEED_INIT(grp) \ 455c9de560dSAlex Tomas (test_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &((grp)->bb_state))) 456c9de560dSAlex Tomas 457c9de560dSAlex Tomas 458c9de560dSAlex Tomas struct ext4_prealloc_space { 459c9de560dSAlex Tomas struct list_head pa_inode_list; 460c9de560dSAlex Tomas struct list_head pa_group_list; 461c9de560dSAlex Tomas union { 462c9de560dSAlex Tomas struct list_head pa_tmp_list; 463c9de560dSAlex Tomas struct rcu_head pa_rcu; 464c9de560dSAlex Tomas } u; 465c9de560dSAlex Tomas spinlock_t pa_lock; 466c9de560dSAlex Tomas atomic_t pa_count; 467c9de560dSAlex Tomas unsigned pa_deleted; 468c9de560dSAlex Tomas ext4_fsblk_t pa_pstart; /* phys. block */ 469c9de560dSAlex Tomas ext4_lblk_t pa_lstart; /* log. block */ 470c9de560dSAlex Tomas unsigned short pa_len; /* len of preallocated chunk */ 471c9de560dSAlex Tomas unsigned short pa_free; /* how many blocks are free */ 472c9de560dSAlex Tomas unsigned short pa_linear; /* consumed in one direction 473c9de560dSAlex Tomas * strictly, for grp prealloc */ 474c9de560dSAlex Tomas spinlock_t *pa_obj_lock; 475c9de560dSAlex Tomas struct inode *pa_inode; /* hack, for history only */ 476c9de560dSAlex Tomas }; 477c9de560dSAlex Tomas 478c9de560dSAlex Tomas 479c9de560dSAlex Tomas struct ext4_free_extent { 480c9de560dSAlex Tomas ext4_lblk_t fe_logical; 481c9de560dSAlex Tomas ext4_grpblk_t fe_start; 482c9de560dSAlex Tomas ext4_group_t fe_group; 483c9de560dSAlex Tomas int fe_len; 484c9de560dSAlex Tomas }; 485c9de560dSAlex Tomas 486c9de560dSAlex Tomas /* 487c9de560dSAlex Tomas * Locality group: 488c9de560dSAlex Tomas * we try to group all related changes together 489c9de560dSAlex Tomas * so that writeback can flush/allocate them together as well 490c9de560dSAlex Tomas */ 491c9de560dSAlex Tomas struct ext4_locality_group { 492c9de560dSAlex Tomas /* for allocator */ 493c9de560dSAlex Tomas struct mutex lg_mutex; /* to serialize allocates */ 494c9de560dSAlex Tomas struct list_head lg_prealloc_list;/* list of preallocations */ 495c9de560dSAlex Tomas spinlock_t lg_prealloc_lock; 496c9de560dSAlex Tomas }; 497c9de560dSAlex Tomas 498c9de560dSAlex Tomas struct ext4_allocation_context { 499c9de560dSAlex Tomas struct inode *ac_inode; 500c9de560dSAlex Tomas struct super_block *ac_sb; 501c9de560dSAlex Tomas 502c9de560dSAlex Tomas /* original request */ 503c9de560dSAlex Tomas struct ext4_free_extent ac_o_ex; 504c9de560dSAlex Tomas 505c9de560dSAlex Tomas /* goal request (after normalization) */ 506c9de560dSAlex Tomas struct ext4_free_extent ac_g_ex; 507c9de560dSAlex Tomas 508c9de560dSAlex Tomas /* the best found extent */ 509c9de560dSAlex Tomas struct ext4_free_extent ac_b_ex; 510c9de560dSAlex Tomas 511c9de560dSAlex Tomas /* copy of the bext found extent taken before preallocation efforts */ 512c9de560dSAlex Tomas struct ext4_free_extent ac_f_ex; 513c9de560dSAlex Tomas 514c9de560dSAlex Tomas /* number of iterations done. we have to track to limit searching */ 515c9de560dSAlex Tomas unsigned long ac_ex_scanned; 516c9de560dSAlex Tomas __u16 ac_groups_scanned; 517c9de560dSAlex Tomas __u16 ac_found; 518c9de560dSAlex Tomas __u16 ac_tail; 519c9de560dSAlex Tomas __u16 ac_buddy; 520c9de560dSAlex Tomas __u16 ac_flags; /* allocation hints */ 521c9de560dSAlex Tomas __u8 ac_status; 522c9de560dSAlex Tomas __u8 ac_criteria; 523c9de560dSAlex Tomas __u8 ac_repeats; 524c9de560dSAlex Tomas __u8 ac_2order; /* if request is to allocate 2^N blocks and 525c9de560dSAlex Tomas * N > 0, the field stores N, otherwise 0 */ 526c9de560dSAlex Tomas __u8 ac_op; /* operation, for history only */ 527c9de560dSAlex Tomas struct page *ac_bitmap_page; 528c9de560dSAlex Tomas struct page *ac_buddy_page; 529c9de560dSAlex Tomas struct ext4_prealloc_space *ac_pa; 530c9de560dSAlex Tomas struct ext4_locality_group *ac_lg; 531c9de560dSAlex Tomas }; 532c9de560dSAlex Tomas 533c9de560dSAlex Tomas #define AC_STATUS_CONTINUE 1 534c9de560dSAlex Tomas #define AC_STATUS_FOUND 2 535c9de560dSAlex Tomas #define AC_STATUS_BREAK 3 536c9de560dSAlex Tomas 537c9de560dSAlex Tomas struct ext4_mb_history { 538c9de560dSAlex Tomas struct ext4_free_extent orig; /* orig allocation */ 539c9de560dSAlex Tomas struct ext4_free_extent goal; /* goal allocation */ 540c9de560dSAlex Tomas struct ext4_free_extent result; /* result allocation */ 541c9de560dSAlex Tomas unsigned pid; 542c9de560dSAlex Tomas unsigned ino; 543c9de560dSAlex Tomas __u16 found; /* how many extents have been found */ 544c9de560dSAlex Tomas __u16 groups; /* how many groups have been scanned */ 545c9de560dSAlex Tomas __u16 tail; /* what tail broke some buddy */ 546c9de560dSAlex Tomas __u16 buddy; /* buddy the tail ^^^ broke */ 547c9de560dSAlex Tomas __u16 flags; 548c9de560dSAlex Tomas __u8 cr:3; /* which phase the result extent was found at */ 549c9de560dSAlex Tomas __u8 op:4; 550c9de560dSAlex Tomas __u8 merged:1; 551c9de560dSAlex Tomas }; 552c9de560dSAlex Tomas 553c9de560dSAlex Tomas struct ext4_buddy { 554c9de560dSAlex Tomas struct page *bd_buddy_page; 555c9de560dSAlex Tomas void *bd_buddy; 556c9de560dSAlex Tomas struct page *bd_bitmap_page; 557c9de560dSAlex Tomas void *bd_bitmap; 558c9de560dSAlex Tomas struct ext4_group_info *bd_info; 559c9de560dSAlex Tomas struct super_block *bd_sb; 560c9de560dSAlex Tomas __u16 bd_blkbits; 561c9de560dSAlex Tomas ext4_group_t bd_group; 562c9de560dSAlex Tomas }; 563c9de560dSAlex Tomas #define EXT4_MB_BITMAP(e4b) ((e4b)->bd_bitmap) 564c9de560dSAlex Tomas #define EXT4_MB_BUDDY(e4b) ((e4b)->bd_buddy) 565c9de560dSAlex Tomas 566c9de560dSAlex Tomas #ifndef EXT4_MB_HISTORY 567c9de560dSAlex Tomas static inline void ext4_mb_store_history(struct ext4_allocation_context *ac) 568c9de560dSAlex Tomas { 569c9de560dSAlex Tomas return; 570c9de560dSAlex Tomas } 571c9de560dSAlex Tomas #else 572c9de560dSAlex Tomas static void ext4_mb_store_history(struct ext4_allocation_context *ac); 573c9de560dSAlex Tomas #endif 574c9de560dSAlex Tomas 575c9de560dSAlex Tomas #define in_range(b, first, len) ((b) >= (first) && (b) <= (first) + (len) - 1) 576c9de560dSAlex Tomas 577c9de560dSAlex Tomas static struct proc_dir_entry *proc_root_ext4; 578c9de560dSAlex Tomas struct buffer_head *read_block_bitmap(struct super_block *, ext4_group_t); 579c9de560dSAlex Tomas ext4_fsblk_t ext4_new_blocks_old(handle_t *handle, struct inode *inode, 580c9de560dSAlex Tomas ext4_fsblk_t goal, unsigned long *count, int *errp); 581c9de560dSAlex Tomas 582c9de560dSAlex Tomas static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 583c9de560dSAlex Tomas ext4_group_t group); 584c9de560dSAlex Tomas static void ext4_mb_poll_new_transaction(struct super_block *, handle_t *); 585c9de560dSAlex Tomas static void ext4_mb_free_committed_blocks(struct super_block *); 586c9de560dSAlex Tomas static void ext4_mb_return_to_preallocation(struct inode *inode, 587c9de560dSAlex Tomas struct ext4_buddy *e4b, sector_t block, 588c9de560dSAlex Tomas int count); 589c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *, 590c9de560dSAlex Tomas struct super_block *, struct ext4_prealloc_space *pa); 591c9de560dSAlex Tomas static int ext4_mb_init_per_dev_proc(struct super_block *sb); 592c9de560dSAlex Tomas static int ext4_mb_destroy_per_dev_proc(struct super_block *sb); 593c9de560dSAlex Tomas 594c9de560dSAlex Tomas 595c9de560dSAlex Tomas static inline void ext4_lock_group(struct super_block *sb, ext4_group_t group) 596c9de560dSAlex Tomas { 597c9de560dSAlex Tomas struct ext4_group_info *grinfo = ext4_get_group_info(sb, group); 598c9de560dSAlex Tomas 599c9de560dSAlex Tomas bit_spin_lock(EXT4_GROUP_INFO_LOCKED_BIT, &(grinfo->bb_state)); 600c9de560dSAlex Tomas } 601c9de560dSAlex Tomas 602c9de560dSAlex Tomas static inline void ext4_unlock_group(struct super_block *sb, 603c9de560dSAlex Tomas ext4_group_t group) 604c9de560dSAlex Tomas { 605c9de560dSAlex Tomas struct ext4_group_info *grinfo = ext4_get_group_info(sb, group); 606c9de560dSAlex Tomas 607c9de560dSAlex Tomas bit_spin_unlock(EXT4_GROUP_INFO_LOCKED_BIT, &(grinfo->bb_state)); 608c9de560dSAlex Tomas } 609c9de560dSAlex Tomas 610c9de560dSAlex Tomas static inline int ext4_is_group_locked(struct super_block *sb, 611c9de560dSAlex Tomas ext4_group_t group) 612c9de560dSAlex Tomas { 613c9de560dSAlex Tomas struct ext4_group_info *grinfo = ext4_get_group_info(sb, group); 614c9de560dSAlex Tomas 615c9de560dSAlex Tomas return bit_spin_is_locked(EXT4_GROUP_INFO_LOCKED_BIT, 616c9de560dSAlex Tomas &(grinfo->bb_state)); 617c9de560dSAlex Tomas } 618c9de560dSAlex Tomas 619c9de560dSAlex Tomas static ext4_fsblk_t ext4_grp_offs_to_block(struct super_block *sb, 620c9de560dSAlex Tomas struct ext4_free_extent *fex) 621c9de560dSAlex Tomas { 622c9de560dSAlex Tomas ext4_fsblk_t block; 623c9de560dSAlex Tomas 624c9de560dSAlex Tomas block = (ext4_fsblk_t) fex->fe_group * EXT4_BLOCKS_PER_GROUP(sb) 625c9de560dSAlex Tomas + fex->fe_start 626c9de560dSAlex Tomas + le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 627c9de560dSAlex Tomas return block; 628c9de560dSAlex Tomas } 629c9de560dSAlex Tomas 630c9de560dSAlex Tomas #if BITS_PER_LONG == 64 631c9de560dSAlex Tomas #define mb_correct_addr_and_bit(bit, addr) \ 632c9de560dSAlex Tomas { \ 633c9de560dSAlex Tomas bit += ((unsigned long) addr & 7UL) << 3; \ 634c9de560dSAlex Tomas addr = (void *) ((unsigned long) addr & ~7UL); \ 635c9de560dSAlex Tomas } 636c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 637c9de560dSAlex Tomas #define mb_correct_addr_and_bit(bit, addr) \ 638c9de560dSAlex Tomas { \ 639c9de560dSAlex Tomas bit += ((unsigned long) addr & 3UL) << 3; \ 640c9de560dSAlex Tomas addr = (void *) ((unsigned long) addr & ~3UL); \ 641c9de560dSAlex Tomas } 642c9de560dSAlex Tomas #else 643c9de560dSAlex Tomas #error "how many bits you are?!" 644c9de560dSAlex Tomas #endif 645c9de560dSAlex Tomas 646c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 647c9de560dSAlex Tomas { 648c9de560dSAlex Tomas /* 649c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 650c9de560dSAlex Tomas * needs unsigned long aligned address 651c9de560dSAlex Tomas */ 652c9de560dSAlex Tomas mb_correct_addr_and_bit(bit, addr); 653c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 654c9de560dSAlex Tomas } 655c9de560dSAlex Tomas 656c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 657c9de560dSAlex Tomas { 658c9de560dSAlex Tomas mb_correct_addr_and_bit(bit, addr); 659c9de560dSAlex Tomas ext4_set_bit(bit, addr); 660c9de560dSAlex Tomas } 661c9de560dSAlex Tomas 662c9de560dSAlex Tomas static inline void mb_set_bit_atomic(spinlock_t *lock, int bit, void *addr) 663c9de560dSAlex Tomas { 664c9de560dSAlex Tomas mb_correct_addr_and_bit(bit, addr); 665c9de560dSAlex Tomas ext4_set_bit_atomic(lock, bit, addr); 666c9de560dSAlex Tomas } 667c9de560dSAlex Tomas 668c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 669c9de560dSAlex Tomas { 670c9de560dSAlex Tomas mb_correct_addr_and_bit(bit, addr); 671c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 672c9de560dSAlex Tomas } 673c9de560dSAlex Tomas 674c9de560dSAlex Tomas static inline void mb_clear_bit_atomic(spinlock_t *lock, int bit, void *addr) 675c9de560dSAlex Tomas { 676c9de560dSAlex Tomas mb_correct_addr_and_bit(bit, addr); 677c9de560dSAlex Tomas ext4_clear_bit_atomic(lock, bit, addr); 678c9de560dSAlex Tomas } 679c9de560dSAlex Tomas 680c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 681c9de560dSAlex Tomas { 682c9de560dSAlex Tomas char *bb; 683c9de560dSAlex Tomas 684c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 685c9de560dSAlex Tomas BUG_ON(max == NULL); 686c9de560dSAlex Tomas 687c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 688c9de560dSAlex Tomas *max = 0; 689c9de560dSAlex Tomas return NULL; 690c9de560dSAlex Tomas } 691c9de560dSAlex Tomas 692c9de560dSAlex Tomas /* at order 0 we see each particular block */ 693c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 694c9de560dSAlex Tomas if (order == 0) 695c9de560dSAlex Tomas return EXT4_MB_BITMAP(e4b); 696c9de560dSAlex Tomas 697c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b) + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 698c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 699c9de560dSAlex Tomas 700c9de560dSAlex Tomas return bb; 701c9de560dSAlex Tomas } 702c9de560dSAlex Tomas 703c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 704c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 705c9de560dSAlex Tomas int first, int count) 706c9de560dSAlex Tomas { 707c9de560dSAlex Tomas int i; 708c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 709c9de560dSAlex Tomas 710c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 711c9de560dSAlex Tomas return; 712c9de560dSAlex Tomas BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group)); 713c9de560dSAlex Tomas for (i = 0; i < count; i++) { 714c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 715c9de560dSAlex Tomas ext4_fsblk_t blocknr; 716c9de560dSAlex Tomas blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb); 717c9de560dSAlex Tomas blocknr += first + i; 718c9de560dSAlex Tomas blocknr += 719c9de560dSAlex Tomas le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 720c9de560dSAlex Tomas 721c9de560dSAlex Tomas ext4_error(sb, __FUNCTION__, "double-free of inode" 722c9de560dSAlex Tomas " %lu's block %llu(bit %u in group %lu)\n", 723c9de560dSAlex Tomas inode ? inode->i_ino : 0, blocknr, 724c9de560dSAlex Tomas first + i, e4b->bd_group); 725c9de560dSAlex Tomas } 726c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 727c9de560dSAlex Tomas } 728c9de560dSAlex Tomas } 729c9de560dSAlex Tomas 730c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 731c9de560dSAlex Tomas { 732c9de560dSAlex Tomas int i; 733c9de560dSAlex Tomas 734c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 735c9de560dSAlex Tomas return; 736c9de560dSAlex Tomas BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); 737c9de560dSAlex Tomas for (i = 0; i < count; i++) { 738c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 739c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 740c9de560dSAlex Tomas } 741c9de560dSAlex Tomas } 742c9de560dSAlex Tomas 743c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 744c9de560dSAlex Tomas { 745c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 746c9de560dSAlex Tomas unsigned char *b1, *b2; 747c9de560dSAlex Tomas int i; 748c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 749c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 750c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 751c9de560dSAlex Tomas if (b1[i] != b2[i]) { 752c9de560dSAlex Tomas printk("corruption in group %lu at byte %u(%u):" 753c9de560dSAlex Tomas " %x in copy != %x on disk/prealloc\n", 754c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 755c9de560dSAlex Tomas BUG(); 756c9de560dSAlex Tomas } 757c9de560dSAlex Tomas } 758c9de560dSAlex Tomas } 759c9de560dSAlex Tomas } 760c9de560dSAlex Tomas 761c9de560dSAlex Tomas #else 762c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 763c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 764c9de560dSAlex Tomas { 765c9de560dSAlex Tomas return; 766c9de560dSAlex Tomas } 767c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 768c9de560dSAlex Tomas int first, int count) 769c9de560dSAlex Tomas { 770c9de560dSAlex Tomas return; 771c9de560dSAlex Tomas } 772c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 773c9de560dSAlex Tomas { 774c9de560dSAlex Tomas return; 775c9de560dSAlex Tomas } 776c9de560dSAlex Tomas #endif 777c9de560dSAlex Tomas 778c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 779c9de560dSAlex Tomas 780c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 781c9de560dSAlex Tomas do { \ 782c9de560dSAlex Tomas if (!(assert)) { \ 783c9de560dSAlex Tomas printk(KERN_EMERG \ 784c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 785c9de560dSAlex Tomas function, file, line, # assert); \ 786c9de560dSAlex Tomas BUG(); \ 787c9de560dSAlex Tomas } \ 788c9de560dSAlex Tomas } while (0) 789c9de560dSAlex Tomas 790c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 791c9de560dSAlex Tomas const char *function, int line) 792c9de560dSAlex Tomas { 793c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 794c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 795c9de560dSAlex Tomas int max; 796c9de560dSAlex Tomas int max2; 797c9de560dSAlex Tomas int i; 798c9de560dSAlex Tomas int j; 799c9de560dSAlex Tomas int k; 800c9de560dSAlex Tomas int count; 801c9de560dSAlex Tomas struct ext4_group_info *grp; 802c9de560dSAlex Tomas int fragments = 0; 803c9de560dSAlex Tomas int fstart; 804c9de560dSAlex Tomas struct list_head *cur; 805c9de560dSAlex Tomas void *buddy; 806c9de560dSAlex Tomas void *buddy2; 807c9de560dSAlex Tomas 808c9de560dSAlex Tomas if (!test_opt(sb, MBALLOC)) 809c9de560dSAlex Tomas return 0; 810c9de560dSAlex Tomas 811c9de560dSAlex Tomas { 812c9de560dSAlex Tomas static int mb_check_counter; 813c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 814c9de560dSAlex Tomas return 0; 815c9de560dSAlex Tomas } 816c9de560dSAlex Tomas 817c9de560dSAlex Tomas while (order > 1) { 818c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 819c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 820c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 821c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 822c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 823c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 824c9de560dSAlex Tomas 825c9de560dSAlex Tomas count = 0; 826c9de560dSAlex Tomas for (i = 0; i < max; i++) { 827c9de560dSAlex Tomas 828c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 829c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 830c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 831c9de560dSAlex Tomas MB_CHECK_ASSERT( 832c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 833c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 834c9de560dSAlex Tomas MB_CHECK_ASSERT( 835c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 836c9de560dSAlex Tomas } 837c9de560dSAlex Tomas continue; 838c9de560dSAlex Tomas } 839c9de560dSAlex Tomas 840c9de560dSAlex Tomas /* both bits in buddy2 must be 0 */ 841c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 842c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 843c9de560dSAlex Tomas 844c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 845c9de560dSAlex Tomas k = (i * (1 << order)) + j; 846c9de560dSAlex Tomas MB_CHECK_ASSERT( 847c9de560dSAlex Tomas !mb_test_bit(k, EXT4_MB_BITMAP(e4b))); 848c9de560dSAlex Tomas } 849c9de560dSAlex Tomas count++; 850c9de560dSAlex Tomas } 851c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 852c9de560dSAlex Tomas order--; 853c9de560dSAlex Tomas } 854c9de560dSAlex Tomas 855c9de560dSAlex Tomas fstart = -1; 856c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 857c9de560dSAlex Tomas for (i = 0; i < max; i++) { 858c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 859c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 860c9de560dSAlex Tomas if (fstart == -1) { 861c9de560dSAlex Tomas fragments++; 862c9de560dSAlex Tomas fstart = i; 863c9de560dSAlex Tomas } 864c9de560dSAlex Tomas continue; 865c9de560dSAlex Tomas } 866c9de560dSAlex Tomas fstart = -1; 867c9de560dSAlex Tomas /* check used bits only */ 868c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 869c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 870c9de560dSAlex Tomas k = i >> j; 871c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 872c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 873c9de560dSAlex Tomas } 874c9de560dSAlex Tomas } 875c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 876c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 877c9de560dSAlex Tomas 878c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 879c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 880c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 881c9de560dSAlex Tomas ext4_group_t groupnr; 882c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 883c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, group_list); 884c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pstart, &groupnr, &k); 885c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 886c9de560dSAlex Tomas for (i = 0; i < pa->len; i++) 887c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 888c9de560dSAlex Tomas } 889c9de560dSAlex Tomas return 0; 890c9de560dSAlex Tomas } 891c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 892c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 893c9de560dSAlex Tomas __FILE__, __FUNCTION__, __LINE__) 894c9de560dSAlex Tomas #else 895c9de560dSAlex Tomas #define mb_check_buddy(e4b) 896c9de560dSAlex Tomas #endif 897c9de560dSAlex Tomas 898c9de560dSAlex Tomas /* FIXME!! need more doc */ 899c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 900c9de560dSAlex Tomas void *buddy, unsigned first, int len, 901c9de560dSAlex Tomas struct ext4_group_info *grp) 902c9de560dSAlex Tomas { 903c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 904c9de560dSAlex Tomas unsigned short min; 905c9de560dSAlex Tomas unsigned short max; 906c9de560dSAlex Tomas unsigned short chunk; 907c9de560dSAlex Tomas unsigned short border; 908c9de560dSAlex Tomas 909b73fce69SValerie Clement BUG_ON(len > EXT4_BLOCKS_PER_GROUP(sb)); 910c9de560dSAlex Tomas 911c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 912c9de560dSAlex Tomas 913c9de560dSAlex Tomas while (len > 0) { 914c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 915c9de560dSAlex Tomas max = ffs(first | border) - 1; 916c9de560dSAlex Tomas 917c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 918c9de560dSAlex Tomas min = fls(len) - 1; 919c9de560dSAlex Tomas 920c9de560dSAlex Tomas if (max < min) 921c9de560dSAlex Tomas min = max; 922c9de560dSAlex Tomas chunk = 1 << min; 923c9de560dSAlex Tomas 924c9de560dSAlex Tomas /* mark multiblock chunks only */ 925c9de560dSAlex Tomas grp->bb_counters[min]++; 926c9de560dSAlex Tomas if (min > 0) 927c9de560dSAlex Tomas mb_clear_bit(first >> min, 928c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 929c9de560dSAlex Tomas 930c9de560dSAlex Tomas len -= chunk; 931c9de560dSAlex Tomas first += chunk; 932c9de560dSAlex Tomas } 933c9de560dSAlex Tomas } 934c9de560dSAlex Tomas 935c9de560dSAlex Tomas static void ext4_mb_generate_buddy(struct super_block *sb, 936c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 937c9de560dSAlex Tomas { 938c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 939c9de560dSAlex Tomas unsigned short max = EXT4_BLOCKS_PER_GROUP(sb); 940c9de560dSAlex Tomas unsigned short i = 0; 941c9de560dSAlex Tomas unsigned short first; 942c9de560dSAlex Tomas unsigned short len; 943c9de560dSAlex Tomas unsigned free = 0; 944c9de560dSAlex Tomas unsigned fragments = 0; 945c9de560dSAlex Tomas unsigned long long period = get_cycles(); 946c9de560dSAlex Tomas 947c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 948c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 949c9de560dSAlex Tomas i = ext4_find_next_zero_bit(bitmap, max, 0); 950c9de560dSAlex Tomas grp->bb_first_free = i; 951c9de560dSAlex Tomas while (i < max) { 952c9de560dSAlex Tomas fragments++; 953c9de560dSAlex Tomas first = i; 954c9de560dSAlex Tomas i = ext4_find_next_bit(bitmap, max, i); 955c9de560dSAlex Tomas len = i - first; 956c9de560dSAlex Tomas free += len; 957c9de560dSAlex Tomas if (len > 1) 958c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 959c9de560dSAlex Tomas else 960c9de560dSAlex Tomas grp->bb_counters[0]++; 961c9de560dSAlex Tomas if (i < max) 962c9de560dSAlex Tomas i = ext4_find_next_zero_bit(bitmap, max, i); 963c9de560dSAlex Tomas } 964c9de560dSAlex Tomas grp->bb_fragments = fragments; 965c9de560dSAlex Tomas 966c9de560dSAlex Tomas if (free != grp->bb_free) { 96726346ff6SAneesh Kumar K.V ext4_error(sb, __FUNCTION__, 968c9de560dSAlex Tomas "EXT4-fs: group %lu: %u blocks in bitmap, %u in gd\n", 969c9de560dSAlex Tomas group, free, grp->bb_free); 970e56eb659SAneesh Kumar K.V /* 971e56eb659SAneesh Kumar K.V * If we intent to continue, we consider group descritor 972e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 973e56eb659SAneesh Kumar K.V */ 974c9de560dSAlex Tomas grp->bb_free = free; 975c9de560dSAlex Tomas } 976c9de560dSAlex Tomas 977c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 978c9de560dSAlex Tomas 979c9de560dSAlex Tomas period = get_cycles() - period; 980c9de560dSAlex Tomas spin_lock(&EXT4_SB(sb)->s_bal_lock); 981c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_buddies_generated++; 982c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_generation_time += period; 983c9de560dSAlex Tomas spin_unlock(&EXT4_SB(sb)->s_bal_lock); 984c9de560dSAlex Tomas } 985c9de560dSAlex Tomas 986c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 987c9de560dSAlex Tomas * for convenience. The information regarding each group 988c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 989c9de560dSAlex Tomas * block bitmap and buddy information. The information are 990c9de560dSAlex Tomas * stored in the inode as 991c9de560dSAlex Tomas * 992c9de560dSAlex Tomas * { page } 993c9de560dSAlex Tomas * [ group 0 buddy][ group 0 bitmap] [group 1][ group 1]... 994c9de560dSAlex Tomas * 995c9de560dSAlex Tomas * 996c9de560dSAlex Tomas * one block each for bitmap and buddy information. 997c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 998c9de560dSAlex Tomas * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks. 999c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 1000c9de560dSAlex Tomas * is blocks_per_page/2 1001c9de560dSAlex Tomas */ 1002c9de560dSAlex Tomas 1003c9de560dSAlex Tomas static int ext4_mb_init_cache(struct page *page, char *incore) 1004c9de560dSAlex Tomas { 1005c9de560dSAlex Tomas int blocksize; 1006c9de560dSAlex Tomas int blocks_per_page; 1007c9de560dSAlex Tomas int groups_per_page; 1008c9de560dSAlex Tomas int err = 0; 1009c9de560dSAlex Tomas int i; 1010c9de560dSAlex Tomas ext4_group_t first_group; 1011c9de560dSAlex Tomas int first_block; 1012c9de560dSAlex Tomas struct super_block *sb; 1013c9de560dSAlex Tomas struct buffer_head *bhs; 1014c9de560dSAlex Tomas struct buffer_head **bh; 1015c9de560dSAlex Tomas struct inode *inode; 1016c9de560dSAlex Tomas char *data; 1017c9de560dSAlex Tomas char *bitmap; 1018c9de560dSAlex Tomas 1019c9de560dSAlex Tomas mb_debug("init page %lu\n", page->index); 1020c9de560dSAlex Tomas 1021c9de560dSAlex Tomas inode = page->mapping->host; 1022c9de560dSAlex Tomas sb = inode->i_sb; 1023c9de560dSAlex Tomas blocksize = 1 << inode->i_blkbits; 1024c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / blocksize; 1025c9de560dSAlex Tomas 1026c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 1027c9de560dSAlex Tomas if (groups_per_page == 0) 1028c9de560dSAlex Tomas groups_per_page = 1; 1029c9de560dSAlex Tomas 1030c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 1031c9de560dSAlex Tomas if (groups_per_page > 1) { 1032c9de560dSAlex Tomas err = -ENOMEM; 1033c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 1034c9de560dSAlex Tomas bh = kzalloc(i, GFP_NOFS); 1035c9de560dSAlex Tomas if (bh == NULL) 1036c9de560dSAlex Tomas goto out; 1037c9de560dSAlex Tomas } else 1038c9de560dSAlex Tomas bh = &bhs; 1039c9de560dSAlex Tomas 1040c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 1041c9de560dSAlex Tomas 1042c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 1043c9de560dSAlex Tomas for (i = 0; i < groups_per_page; i++) { 1044c9de560dSAlex Tomas struct ext4_group_desc *desc; 1045c9de560dSAlex Tomas 1046c9de560dSAlex Tomas if (first_group + i >= EXT4_SB(sb)->s_groups_count) 1047c9de560dSAlex Tomas break; 1048c9de560dSAlex Tomas 1049c9de560dSAlex Tomas err = -EIO; 1050c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, first_group + i, NULL); 1051c9de560dSAlex Tomas if (desc == NULL) 1052c9de560dSAlex Tomas goto out; 1053c9de560dSAlex Tomas 1054c9de560dSAlex Tomas err = -ENOMEM; 1055c9de560dSAlex Tomas bh[i] = sb_getblk(sb, ext4_block_bitmap(sb, desc)); 1056c9de560dSAlex Tomas if (bh[i] == NULL) 1057c9de560dSAlex Tomas goto out; 1058c9de560dSAlex Tomas 1059c9de560dSAlex Tomas if (bh_uptodate_or_lock(bh[i])) 1060c9de560dSAlex Tomas continue; 1061c9de560dSAlex Tomas 1062c9de560dSAlex Tomas if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 1063c9de560dSAlex Tomas ext4_init_block_bitmap(sb, bh[i], 1064c9de560dSAlex Tomas first_group + i, desc); 1065c9de560dSAlex Tomas set_buffer_uptodate(bh[i]); 1066c9de560dSAlex Tomas unlock_buffer(bh[i]); 1067c9de560dSAlex Tomas continue; 1068c9de560dSAlex Tomas } 1069c9de560dSAlex Tomas get_bh(bh[i]); 1070c9de560dSAlex Tomas bh[i]->b_end_io = end_buffer_read_sync; 1071c9de560dSAlex Tomas submit_bh(READ, bh[i]); 1072c9de560dSAlex Tomas mb_debug("read bitmap for group %lu\n", first_group + i); 1073c9de560dSAlex Tomas } 1074c9de560dSAlex Tomas 1075c9de560dSAlex Tomas /* wait for I/O completion */ 1076c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 1077c9de560dSAlex Tomas wait_on_buffer(bh[i]); 1078c9de560dSAlex Tomas 1079c9de560dSAlex Tomas err = -EIO; 1080c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 1081c9de560dSAlex Tomas if (!buffer_uptodate(bh[i])) 1082c9de560dSAlex Tomas goto out; 1083c9de560dSAlex Tomas 1084c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 1085c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 1086c9de560dSAlex Tomas int group; 1087c9de560dSAlex Tomas struct ext4_group_info *grinfo; 1088c9de560dSAlex Tomas 1089c9de560dSAlex Tomas group = (first_block + i) >> 1; 1090c9de560dSAlex Tomas if (group >= EXT4_SB(sb)->s_groups_count) 1091c9de560dSAlex Tomas break; 1092c9de560dSAlex Tomas 1093c9de560dSAlex Tomas /* 1094c9de560dSAlex Tomas * data carry information regarding this 1095c9de560dSAlex Tomas * particular group in the format specified 1096c9de560dSAlex Tomas * above 1097c9de560dSAlex Tomas * 1098c9de560dSAlex Tomas */ 1099c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 1100c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 1101c9de560dSAlex Tomas 1102c9de560dSAlex Tomas /* 1103c9de560dSAlex Tomas * We place the buddy block and bitmap block 1104c9de560dSAlex Tomas * close together 1105c9de560dSAlex Tomas */ 1106c9de560dSAlex Tomas if ((first_block + i) & 1) { 1107c9de560dSAlex Tomas /* this is block of buddy */ 1108c9de560dSAlex Tomas BUG_ON(incore == NULL); 1109c9de560dSAlex Tomas mb_debug("put buddy for group %u in page %lu/%x\n", 1110c9de560dSAlex Tomas group, page->index, i * blocksize); 1111c9de560dSAlex Tomas memset(data, 0xff, blocksize); 1112c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 1113c9de560dSAlex Tomas grinfo->bb_fragments = 0; 1114c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 1115c9de560dSAlex Tomas sizeof(unsigned short)*(sb->s_blocksize_bits+2)); 1116c9de560dSAlex Tomas /* 1117c9de560dSAlex Tomas * incore got set to the group block bitmap below 1118c9de560dSAlex Tomas */ 1119c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 1120c9de560dSAlex Tomas incore = NULL; 1121c9de560dSAlex Tomas } else { 1122c9de560dSAlex Tomas /* this is block of bitmap */ 1123c9de560dSAlex Tomas BUG_ON(incore != NULL); 1124c9de560dSAlex Tomas mb_debug("put bitmap for group %u in page %lu/%x\n", 1125c9de560dSAlex Tomas group, page->index, i * blocksize); 1126c9de560dSAlex Tomas 1127c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 1128c9de560dSAlex Tomas ext4_lock_group(sb, group); 1129c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 1130c9de560dSAlex Tomas 1131c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 1132c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 1133c9de560dSAlex Tomas ext4_unlock_group(sb, group); 1134c9de560dSAlex Tomas 1135c9de560dSAlex Tomas /* set incore so that the buddy information can be 1136c9de560dSAlex Tomas * generated using this 1137c9de560dSAlex Tomas */ 1138c9de560dSAlex Tomas incore = data; 1139c9de560dSAlex Tomas } 1140c9de560dSAlex Tomas } 1141c9de560dSAlex Tomas SetPageUptodate(page); 1142c9de560dSAlex Tomas 1143c9de560dSAlex Tomas out: 1144c9de560dSAlex Tomas if (bh) { 1145c9de560dSAlex Tomas for (i = 0; i < groups_per_page && bh[i]; i++) 1146c9de560dSAlex Tomas brelse(bh[i]); 1147c9de560dSAlex Tomas if (bh != &bhs) 1148c9de560dSAlex Tomas kfree(bh); 1149c9de560dSAlex Tomas } 1150c9de560dSAlex Tomas return err; 1151c9de560dSAlex Tomas } 1152c9de560dSAlex Tomas 1153c9de560dSAlex Tomas static int ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1154c9de560dSAlex Tomas struct ext4_buddy *e4b) 1155c9de560dSAlex Tomas { 1156c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 1157c9de560dSAlex Tomas struct inode *inode = sbi->s_buddy_cache; 1158c9de560dSAlex Tomas int blocks_per_page; 1159c9de560dSAlex Tomas int block; 1160c9de560dSAlex Tomas int pnum; 1161c9de560dSAlex Tomas int poff; 1162c9de560dSAlex Tomas struct page *page; 1163c9de560dSAlex Tomas 1164c9de560dSAlex Tomas mb_debug("load group %lu\n", group); 1165c9de560dSAlex Tomas 1166c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1167c9de560dSAlex Tomas 1168c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1169c9de560dSAlex Tomas e4b->bd_info = ext4_get_group_info(sb, group); 1170c9de560dSAlex Tomas e4b->bd_sb = sb; 1171c9de560dSAlex Tomas e4b->bd_group = group; 1172c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1173c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1174c9de560dSAlex Tomas 1175c9de560dSAlex Tomas /* 1176c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1177c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1178c9de560dSAlex Tomas * So for each group we need two blocks. 1179c9de560dSAlex Tomas */ 1180c9de560dSAlex Tomas block = group * 2; 1181c9de560dSAlex Tomas pnum = block / blocks_per_page; 1182c9de560dSAlex Tomas poff = block % blocks_per_page; 1183c9de560dSAlex Tomas 1184c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1185c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 1186c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1187c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1188c9de560dSAlex Tomas if (page) 1189c9de560dSAlex Tomas page_cache_release(page); 1190c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1191c9de560dSAlex Tomas if (page) { 1192c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1193c9de560dSAlex Tomas if (!PageUptodate(page)) { 1194c9de560dSAlex Tomas ext4_mb_init_cache(page, NULL); 1195c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1196c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1197c9de560dSAlex Tomas } 1198c9de560dSAlex Tomas unlock_page(page); 1199c9de560dSAlex Tomas } 1200c9de560dSAlex Tomas } 1201c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) 1202c9de560dSAlex Tomas goto err; 1203c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1204c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1205c9de560dSAlex Tomas mark_page_accessed(page); 1206c9de560dSAlex Tomas 1207c9de560dSAlex Tomas block++; 1208c9de560dSAlex Tomas pnum = block / blocks_per_page; 1209c9de560dSAlex Tomas poff = block % blocks_per_page; 1210c9de560dSAlex Tomas 1211c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1212c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1213c9de560dSAlex Tomas if (page) 1214c9de560dSAlex Tomas page_cache_release(page); 1215c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1216c9de560dSAlex Tomas if (page) { 1217c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1218c9de560dSAlex Tomas if (!PageUptodate(page)) 1219c9de560dSAlex Tomas ext4_mb_init_cache(page, e4b->bd_bitmap); 1220c9de560dSAlex Tomas 1221c9de560dSAlex Tomas unlock_page(page); 1222c9de560dSAlex Tomas } 1223c9de560dSAlex Tomas } 1224c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) 1225c9de560dSAlex Tomas goto err; 1226c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1227c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1228c9de560dSAlex Tomas mark_page_accessed(page); 1229c9de560dSAlex Tomas 1230c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 1231c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 1232c9de560dSAlex Tomas 1233c9de560dSAlex Tomas return 0; 1234c9de560dSAlex Tomas 1235c9de560dSAlex Tomas err: 1236c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1237c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1238c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1239c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1240c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1241c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1242c9de560dSAlex Tomas return -EIO; 1243c9de560dSAlex Tomas } 1244c9de560dSAlex Tomas 1245c9de560dSAlex Tomas static void ext4_mb_release_desc(struct ext4_buddy *e4b) 1246c9de560dSAlex Tomas { 1247c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1248c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1249c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1250c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1251c9de560dSAlex Tomas } 1252c9de560dSAlex Tomas 1253c9de560dSAlex Tomas 1254c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1255c9de560dSAlex Tomas { 1256c9de560dSAlex Tomas int order = 1; 1257c9de560dSAlex Tomas void *bb; 1258c9de560dSAlex Tomas 1259c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 1260c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1261c9de560dSAlex Tomas 1262c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b); 1263c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1264c9de560dSAlex Tomas block = block >> 1; 1265c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1266c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1267c9de560dSAlex Tomas return order; 1268c9de560dSAlex Tomas } 1269c9de560dSAlex Tomas bb += 1 << (e4b->bd_blkbits - order); 1270c9de560dSAlex Tomas order++; 1271c9de560dSAlex Tomas } 1272c9de560dSAlex Tomas return 0; 1273c9de560dSAlex Tomas } 1274c9de560dSAlex Tomas 1275c9de560dSAlex Tomas static void mb_clear_bits(spinlock_t *lock, void *bm, int cur, int len) 1276c9de560dSAlex Tomas { 1277c9de560dSAlex Tomas __u32 *addr; 1278c9de560dSAlex Tomas 1279c9de560dSAlex Tomas len = cur + len; 1280c9de560dSAlex Tomas while (cur < len) { 1281c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1282c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1283c9de560dSAlex Tomas addr = bm + (cur >> 3); 1284c9de560dSAlex Tomas *addr = 0; 1285c9de560dSAlex Tomas cur += 32; 1286c9de560dSAlex Tomas continue; 1287c9de560dSAlex Tomas } 1288c9de560dSAlex Tomas mb_clear_bit_atomic(lock, cur, bm); 1289c9de560dSAlex Tomas cur++; 1290c9de560dSAlex Tomas } 1291c9de560dSAlex Tomas } 1292c9de560dSAlex Tomas 1293c9de560dSAlex Tomas static void mb_set_bits(spinlock_t *lock, void *bm, int cur, int len) 1294c9de560dSAlex Tomas { 1295c9de560dSAlex Tomas __u32 *addr; 1296c9de560dSAlex Tomas 1297c9de560dSAlex Tomas len = cur + len; 1298c9de560dSAlex Tomas while (cur < len) { 1299c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1300c9de560dSAlex Tomas /* fast path: set whole word at once */ 1301c9de560dSAlex Tomas addr = bm + (cur >> 3); 1302c9de560dSAlex Tomas *addr = 0xffffffff; 1303c9de560dSAlex Tomas cur += 32; 1304c9de560dSAlex Tomas continue; 1305c9de560dSAlex Tomas } 1306c9de560dSAlex Tomas mb_set_bit_atomic(lock, cur, bm); 1307c9de560dSAlex Tomas cur++; 1308c9de560dSAlex Tomas } 1309c9de560dSAlex Tomas } 1310c9de560dSAlex Tomas 1311c9de560dSAlex Tomas static int mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1312c9de560dSAlex Tomas int first, int count) 1313c9de560dSAlex Tomas { 1314c9de560dSAlex Tomas int block = 0; 1315c9de560dSAlex Tomas int max = 0; 1316c9de560dSAlex Tomas int order; 1317c9de560dSAlex Tomas void *buddy; 1318c9de560dSAlex Tomas void *buddy2; 1319c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1320c9de560dSAlex Tomas 1321c9de560dSAlex Tomas BUG_ON(first + count > (sb->s_blocksize << 3)); 1322c9de560dSAlex Tomas BUG_ON(!ext4_is_group_locked(sb, e4b->bd_group)); 1323c9de560dSAlex Tomas mb_check_buddy(e4b); 1324c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1325c9de560dSAlex Tomas 1326c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1327c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1328c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1329c9de560dSAlex Tomas 1330c9de560dSAlex Tomas /* let's maintain fragments counter */ 1331c9de560dSAlex Tomas if (first != 0) 1332c9de560dSAlex Tomas block = !mb_test_bit(first - 1, EXT4_MB_BITMAP(e4b)); 1333c9de560dSAlex Tomas if (first + count < EXT4_SB(sb)->s_mb_maxs[0]) 1334c9de560dSAlex Tomas max = !mb_test_bit(first + count, EXT4_MB_BITMAP(e4b)); 1335c9de560dSAlex Tomas if (block && max) 1336c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1337c9de560dSAlex Tomas else if (!block && !max) 1338c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1339c9de560dSAlex Tomas 1340c9de560dSAlex Tomas /* let's maintain buddy itself */ 1341c9de560dSAlex Tomas while (count-- > 0) { 1342c9de560dSAlex Tomas block = first++; 1343c9de560dSAlex Tomas order = 0; 1344c9de560dSAlex Tomas 1345c9de560dSAlex Tomas if (!mb_test_bit(block, EXT4_MB_BITMAP(e4b))) { 1346c9de560dSAlex Tomas ext4_fsblk_t blocknr; 1347c9de560dSAlex Tomas blocknr = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb); 1348c9de560dSAlex Tomas blocknr += block; 1349c9de560dSAlex Tomas blocknr += 1350c9de560dSAlex Tomas le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 1351c9de560dSAlex Tomas 1352c9de560dSAlex Tomas ext4_error(sb, __FUNCTION__, "double-free of inode" 1353c9de560dSAlex Tomas " %lu's block %llu(bit %u in group %lu)\n", 1354c9de560dSAlex Tomas inode ? inode->i_ino : 0, blocknr, block, 1355c9de560dSAlex Tomas e4b->bd_group); 1356c9de560dSAlex Tomas } 1357c9de560dSAlex Tomas mb_clear_bit(block, EXT4_MB_BITMAP(e4b)); 1358c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1359c9de560dSAlex Tomas 1360c9de560dSAlex Tomas /* start of the buddy */ 1361c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1362c9de560dSAlex Tomas 1363c9de560dSAlex Tomas do { 1364c9de560dSAlex Tomas block &= ~1UL; 1365c9de560dSAlex Tomas if (mb_test_bit(block, buddy) || 1366c9de560dSAlex Tomas mb_test_bit(block + 1, buddy)) 1367c9de560dSAlex Tomas break; 1368c9de560dSAlex Tomas 1369c9de560dSAlex Tomas /* both the buddies are free, try to coalesce them */ 1370c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order + 1, &max); 1371c9de560dSAlex Tomas 1372c9de560dSAlex Tomas if (!buddy2) 1373c9de560dSAlex Tomas break; 1374c9de560dSAlex Tomas 1375c9de560dSAlex Tomas if (order > 0) { 1376c9de560dSAlex Tomas /* for special purposes, we don't set 1377c9de560dSAlex Tomas * free bits in bitmap */ 1378c9de560dSAlex Tomas mb_set_bit(block, buddy); 1379c9de560dSAlex Tomas mb_set_bit(block + 1, buddy); 1380c9de560dSAlex Tomas } 1381c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1382c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1383c9de560dSAlex Tomas 1384c9de560dSAlex Tomas block = block >> 1; 1385c9de560dSAlex Tomas order++; 1386c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1387c9de560dSAlex Tomas 1388c9de560dSAlex Tomas mb_clear_bit(block, buddy2); 1389c9de560dSAlex Tomas buddy = buddy2; 1390c9de560dSAlex Tomas } while (1); 1391c9de560dSAlex Tomas } 1392c9de560dSAlex Tomas mb_check_buddy(e4b); 1393c9de560dSAlex Tomas 1394c9de560dSAlex Tomas return 0; 1395c9de560dSAlex Tomas } 1396c9de560dSAlex Tomas 1397c9de560dSAlex Tomas static int mb_find_extent(struct ext4_buddy *e4b, int order, int block, 1398c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1399c9de560dSAlex Tomas { 1400c9de560dSAlex Tomas int next = block; 1401c9de560dSAlex Tomas int max; 1402c9de560dSAlex Tomas int ord; 1403c9de560dSAlex Tomas void *buddy; 1404c9de560dSAlex Tomas 1405c9de560dSAlex Tomas BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); 1406c9de560dSAlex Tomas BUG_ON(ex == NULL); 1407c9de560dSAlex Tomas 1408c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1409c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1410c9de560dSAlex Tomas BUG_ON(block >= max); 1411c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1412c9de560dSAlex Tomas ex->fe_len = 0; 1413c9de560dSAlex Tomas ex->fe_start = 0; 1414c9de560dSAlex Tomas ex->fe_group = 0; 1415c9de560dSAlex Tomas return 0; 1416c9de560dSAlex Tomas } 1417c9de560dSAlex Tomas 1418c9de560dSAlex Tomas /* FIXME dorp order completely ? */ 1419c9de560dSAlex Tomas if (likely(order == 0)) { 1420c9de560dSAlex Tomas /* find actual order */ 1421c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1422c9de560dSAlex Tomas block = block >> order; 1423c9de560dSAlex Tomas } 1424c9de560dSAlex Tomas 1425c9de560dSAlex Tomas ex->fe_len = 1 << order; 1426c9de560dSAlex Tomas ex->fe_start = block << order; 1427c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1428c9de560dSAlex Tomas 1429c9de560dSAlex Tomas /* calc difference from given start */ 1430c9de560dSAlex Tomas next = next - ex->fe_start; 1431c9de560dSAlex Tomas ex->fe_len -= next; 1432c9de560dSAlex Tomas ex->fe_start += next; 1433c9de560dSAlex Tomas 1434c9de560dSAlex Tomas while (needed > ex->fe_len && 1435c9de560dSAlex Tomas (buddy = mb_find_buddy(e4b, order, &max))) { 1436c9de560dSAlex Tomas 1437c9de560dSAlex Tomas if (block + 1 >= max) 1438c9de560dSAlex Tomas break; 1439c9de560dSAlex Tomas 1440c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1441c9de560dSAlex Tomas if (mb_test_bit(next, EXT4_MB_BITMAP(e4b))) 1442c9de560dSAlex Tomas break; 1443c9de560dSAlex Tomas 1444c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, next); 1445c9de560dSAlex Tomas 1446c9de560dSAlex Tomas order = ord; 1447c9de560dSAlex Tomas block = next >> order; 1448c9de560dSAlex Tomas ex->fe_len += 1 << order; 1449c9de560dSAlex Tomas } 1450c9de560dSAlex Tomas 1451c9de560dSAlex Tomas BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3))); 1452c9de560dSAlex Tomas return ex->fe_len; 1453c9de560dSAlex Tomas } 1454c9de560dSAlex Tomas 1455c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1456c9de560dSAlex Tomas { 1457c9de560dSAlex Tomas int ord; 1458c9de560dSAlex Tomas int mlen = 0; 1459c9de560dSAlex Tomas int max = 0; 1460c9de560dSAlex Tomas int cur; 1461c9de560dSAlex Tomas int start = ex->fe_start; 1462c9de560dSAlex Tomas int len = ex->fe_len; 1463c9de560dSAlex Tomas unsigned ret = 0; 1464c9de560dSAlex Tomas int len0 = len; 1465c9de560dSAlex Tomas void *buddy; 1466c9de560dSAlex Tomas 1467c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1468c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1469c9de560dSAlex Tomas BUG_ON(!ext4_is_group_locked(e4b->bd_sb, e4b->bd_group)); 1470c9de560dSAlex Tomas mb_check_buddy(e4b); 1471c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1472c9de560dSAlex Tomas 1473c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1474c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1475c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1476c9de560dSAlex Tomas 1477c9de560dSAlex Tomas /* let's maintain fragments counter */ 1478c9de560dSAlex Tomas if (start != 0) 1479c9de560dSAlex Tomas mlen = !mb_test_bit(start - 1, EXT4_MB_BITMAP(e4b)); 1480c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1481c9de560dSAlex Tomas max = !mb_test_bit(start + len, EXT4_MB_BITMAP(e4b)); 1482c9de560dSAlex Tomas if (mlen && max) 1483c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1484c9de560dSAlex Tomas else if (!mlen && !max) 1485c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1486c9de560dSAlex Tomas 1487c9de560dSAlex Tomas /* let's maintain buddy itself */ 1488c9de560dSAlex Tomas while (len) { 1489c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1490c9de560dSAlex Tomas 1491c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1492c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1493c9de560dSAlex Tomas mlen = 1 << ord; 1494c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1495c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1496c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1497c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1498c9de560dSAlex Tomas start += mlen; 1499c9de560dSAlex Tomas len -= mlen; 1500c9de560dSAlex Tomas BUG_ON(len < 0); 1501c9de560dSAlex Tomas continue; 1502c9de560dSAlex Tomas } 1503c9de560dSAlex Tomas 1504c9de560dSAlex Tomas /* store for history */ 1505c9de560dSAlex Tomas if (ret == 0) 1506c9de560dSAlex Tomas ret = len | (ord << 16); 1507c9de560dSAlex Tomas 1508c9de560dSAlex Tomas /* we have to split large buddy */ 1509c9de560dSAlex Tomas BUG_ON(ord <= 0); 1510c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1511c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1512c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1513c9de560dSAlex Tomas 1514c9de560dSAlex Tomas ord--; 1515c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1516c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1517c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1518c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1519c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1520c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1521c9de560dSAlex Tomas } 1522c9de560dSAlex Tomas 1523c9de560dSAlex Tomas mb_set_bits(sb_bgl_lock(EXT4_SB(e4b->bd_sb), ex->fe_group), 1524c9de560dSAlex Tomas EXT4_MB_BITMAP(e4b), ex->fe_start, len0); 1525c9de560dSAlex Tomas mb_check_buddy(e4b); 1526c9de560dSAlex Tomas 1527c9de560dSAlex Tomas return ret; 1528c9de560dSAlex Tomas } 1529c9de560dSAlex Tomas 1530c9de560dSAlex Tomas /* 1531c9de560dSAlex Tomas * Must be called under group lock! 1532c9de560dSAlex Tomas */ 1533c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1534c9de560dSAlex Tomas struct ext4_buddy *e4b) 1535c9de560dSAlex Tomas { 1536c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1537c9de560dSAlex Tomas int ret; 1538c9de560dSAlex Tomas 1539c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1540c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1541c9de560dSAlex Tomas 1542c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1543c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1544c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1545c9de560dSAlex Tomas 1546c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1547c9de560dSAlex Tomas * allocated blocks for history */ 1548c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1549c9de560dSAlex Tomas 1550c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1551c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1552c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1553c9de560dSAlex Tomas 1554c9de560dSAlex Tomas /* XXXXXXX: SUCH A HORRIBLE **CK */ 1555c9de560dSAlex Tomas /*FIXME!! Why ? */ 1556c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1557c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1558c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1559c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 1560c9de560dSAlex Tomas 1561c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 1562c9de560dSAlex Tomas if ((ac->ac_flags & EXT4_MB_HINT_DATA)) { 1563c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1564c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1565c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1566c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1567c9de560dSAlex Tomas } 1568c9de560dSAlex Tomas } 1569c9de560dSAlex Tomas 1570c9de560dSAlex Tomas /* 1571c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1572c9de560dSAlex Tomas */ 1573c9de560dSAlex Tomas 1574c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1575c9de560dSAlex Tomas struct ext4_buddy *e4b, 1576c9de560dSAlex Tomas int finish_group) 1577c9de560dSAlex Tomas { 1578c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1579c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1580c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1581c9de560dSAlex Tomas struct ext4_free_extent ex; 1582c9de560dSAlex Tomas int max; 1583c9de560dSAlex Tomas 1584c9de560dSAlex Tomas /* 1585c9de560dSAlex Tomas * We don't want to scan for a whole year 1586c9de560dSAlex Tomas */ 1587c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1588c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1589c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1590c9de560dSAlex Tomas return; 1591c9de560dSAlex Tomas } 1592c9de560dSAlex Tomas 1593c9de560dSAlex Tomas /* 1594c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1595c9de560dSAlex Tomas */ 1596c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1597c9de560dSAlex Tomas return; 1598c9de560dSAlex Tomas 1599c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1600c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1601c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1602c9de560dSAlex Tomas * when it was found (within this lock-unlock 1603c9de560dSAlex Tomas * period or not) */ 1604c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex); 1605c9de560dSAlex Tomas if (max >= gex->fe_len) { 1606c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1607c9de560dSAlex Tomas return; 1608c9de560dSAlex Tomas } 1609c9de560dSAlex Tomas } 1610c9de560dSAlex Tomas } 1611c9de560dSAlex Tomas 1612c9de560dSAlex Tomas /* 1613c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1614c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1615c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1616c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1617c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1618c9de560dSAlex Tomas * mballoc can't find good enough extent. 1619c9de560dSAlex Tomas * 1620c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1621c9de560dSAlex Tomas */ 1622c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1623c9de560dSAlex Tomas struct ext4_free_extent *ex, 1624c9de560dSAlex Tomas struct ext4_buddy *e4b) 1625c9de560dSAlex Tomas { 1626c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1627c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1628c9de560dSAlex Tomas 1629c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 1630c9de560dSAlex Tomas BUG_ON(ex->fe_len >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 1631c9de560dSAlex Tomas BUG_ON(ex->fe_start >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 1632c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1633c9de560dSAlex Tomas 1634c9de560dSAlex Tomas ac->ac_found++; 1635c9de560dSAlex Tomas 1636c9de560dSAlex Tomas /* 1637c9de560dSAlex Tomas * The special case - take what you catch first 1638c9de560dSAlex Tomas */ 1639c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1640c9de560dSAlex Tomas *bex = *ex; 1641c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1642c9de560dSAlex Tomas return; 1643c9de560dSAlex Tomas } 1644c9de560dSAlex Tomas 1645c9de560dSAlex Tomas /* 1646c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1647c9de560dSAlex Tomas */ 1648c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1649c9de560dSAlex Tomas *bex = *ex; 1650c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1651c9de560dSAlex Tomas return; 1652c9de560dSAlex Tomas } 1653c9de560dSAlex Tomas 1654c9de560dSAlex Tomas /* 1655c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1656c9de560dSAlex Tomas */ 1657c9de560dSAlex Tomas if (bex->fe_len == 0) { 1658c9de560dSAlex Tomas *bex = *ex; 1659c9de560dSAlex Tomas return; 1660c9de560dSAlex Tomas } 1661c9de560dSAlex Tomas 1662c9de560dSAlex Tomas /* 1663c9de560dSAlex Tomas * If new found extent is better, store it in the context 1664c9de560dSAlex Tomas */ 1665c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1666c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1667c9de560dSAlex Tomas * larger than previous best one is better */ 1668c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1669c9de560dSAlex Tomas *bex = *ex; 1670c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1671c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1672c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1673c9de560dSAlex Tomas * smaller than previous one */ 1674c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1675c9de560dSAlex Tomas *bex = *ex; 1676c9de560dSAlex Tomas } 1677c9de560dSAlex Tomas 1678c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1679c9de560dSAlex Tomas } 1680c9de560dSAlex Tomas 1681c9de560dSAlex Tomas static int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1682c9de560dSAlex Tomas struct ext4_buddy *e4b) 1683c9de560dSAlex Tomas { 1684c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1685c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1686c9de560dSAlex Tomas int max; 1687c9de560dSAlex Tomas int err; 1688c9de560dSAlex Tomas 1689c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1690c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1691c9de560dSAlex Tomas if (err) 1692c9de560dSAlex Tomas return err; 1693c9de560dSAlex Tomas 1694c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1695c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex); 1696c9de560dSAlex Tomas 1697c9de560dSAlex Tomas if (max > 0) { 1698c9de560dSAlex Tomas ac->ac_b_ex = ex; 1699c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1700c9de560dSAlex Tomas } 1701c9de560dSAlex Tomas 1702c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1703c9de560dSAlex Tomas ext4_mb_release_desc(e4b); 1704c9de560dSAlex Tomas 1705c9de560dSAlex Tomas return 0; 1706c9de560dSAlex Tomas } 1707c9de560dSAlex Tomas 1708c9de560dSAlex Tomas static int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1709c9de560dSAlex Tomas struct ext4_buddy *e4b) 1710c9de560dSAlex Tomas { 1711c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1712c9de560dSAlex Tomas int max; 1713c9de560dSAlex Tomas int err; 1714c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1715c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 1716c9de560dSAlex Tomas struct ext4_free_extent ex; 1717c9de560dSAlex Tomas 1718c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1719c9de560dSAlex Tomas return 0; 1720c9de560dSAlex Tomas 1721c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1722c9de560dSAlex Tomas if (err) 1723c9de560dSAlex Tomas return err; 1724c9de560dSAlex Tomas 1725c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1726c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start, 1727c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1728c9de560dSAlex Tomas 1729c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1730c9de560dSAlex Tomas ext4_fsblk_t start; 1731c9de560dSAlex Tomas 1732c9de560dSAlex Tomas start = (e4b->bd_group * EXT4_BLOCKS_PER_GROUP(ac->ac_sb)) + 1733c9de560dSAlex Tomas ex.fe_start + le32_to_cpu(es->s_first_data_block); 1734c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1735c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1736c9de560dSAlex Tomas ac->ac_found++; 1737c9de560dSAlex Tomas ac->ac_b_ex = ex; 1738c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1739c9de560dSAlex Tomas } 1740c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1741c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1742c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1743c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1744c9de560dSAlex Tomas ac->ac_found++; 1745c9de560dSAlex Tomas ac->ac_b_ex = ex; 1746c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1747c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1748c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1749c9de560dSAlex Tomas * number of blocks to an existing extent */ 1750c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1751c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1752c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1753c9de560dSAlex Tomas ac->ac_found++; 1754c9de560dSAlex Tomas ac->ac_b_ex = ex; 1755c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1756c9de560dSAlex Tomas } 1757c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1758c9de560dSAlex Tomas ext4_mb_release_desc(e4b); 1759c9de560dSAlex Tomas 1760c9de560dSAlex Tomas return 0; 1761c9de560dSAlex Tomas } 1762c9de560dSAlex Tomas 1763c9de560dSAlex Tomas /* 1764c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1765c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1766c9de560dSAlex Tomas */ 1767c9de560dSAlex Tomas static void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1768c9de560dSAlex Tomas struct ext4_buddy *e4b) 1769c9de560dSAlex Tomas { 1770c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1771c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1772c9de560dSAlex Tomas void *buddy; 1773c9de560dSAlex Tomas int i; 1774c9de560dSAlex Tomas int k; 1775c9de560dSAlex Tomas int max; 1776c9de560dSAlex Tomas 1777c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1778c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1779c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1780c9de560dSAlex Tomas continue; 1781c9de560dSAlex Tomas 1782c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1783c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1784c9de560dSAlex Tomas 1785c9de560dSAlex Tomas k = ext4_find_next_zero_bit(buddy, max, 0); 1786c9de560dSAlex Tomas BUG_ON(k >= max); 1787c9de560dSAlex Tomas 1788c9de560dSAlex Tomas ac->ac_found++; 1789c9de560dSAlex Tomas 1790c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1791c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1792c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1793c9de560dSAlex Tomas 1794c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1795c9de560dSAlex Tomas 1796c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); 1797c9de560dSAlex Tomas 1798c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1799c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1800c9de560dSAlex Tomas 1801c9de560dSAlex Tomas break; 1802c9de560dSAlex Tomas } 1803c9de560dSAlex Tomas } 1804c9de560dSAlex Tomas 1805c9de560dSAlex Tomas /* 1806c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 1807c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 1808c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 1809c9de560dSAlex Tomas */ 1810c9de560dSAlex Tomas static void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 1811c9de560dSAlex Tomas struct ext4_buddy *e4b) 1812c9de560dSAlex Tomas { 1813c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1814c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1815c9de560dSAlex Tomas struct ext4_free_extent ex; 1816c9de560dSAlex Tomas int i; 1817c9de560dSAlex Tomas int free; 1818c9de560dSAlex Tomas 1819c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 1820c9de560dSAlex Tomas BUG_ON(free <= 0); 1821c9de560dSAlex Tomas 1822c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 1823c9de560dSAlex Tomas 1824c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 1825c9de560dSAlex Tomas i = ext4_find_next_zero_bit(bitmap, 1826c9de560dSAlex Tomas EXT4_BLOCKS_PER_GROUP(sb), i); 1827c9de560dSAlex Tomas if (i >= EXT4_BLOCKS_PER_GROUP(sb)) { 182826346ff6SAneesh Kumar K.V /* 1829e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 183026346ff6SAneesh Kumar K.V * free blocks even though group info says we 183126346ff6SAneesh Kumar K.V * we have free blocks 183226346ff6SAneesh Kumar K.V */ 183326346ff6SAneesh Kumar K.V ext4_error(sb, __FUNCTION__, "%d free blocks as per " 183426346ff6SAneesh Kumar K.V "group info. But bitmap says 0\n", 183526346ff6SAneesh Kumar K.V free); 1836c9de560dSAlex Tomas break; 1837c9de560dSAlex Tomas } 1838c9de560dSAlex Tomas 1839c9de560dSAlex Tomas mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex); 1840c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 184126346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 184226346ff6SAneesh Kumar K.V ext4_error(sb, __FUNCTION__, "%d free blocks as per " 184326346ff6SAneesh Kumar K.V "group info. But got %d blocks\n", 184426346ff6SAneesh Kumar K.V free, ex.fe_len); 1845e56eb659SAneesh Kumar K.V /* 1846e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 1847e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 1848e56eb659SAneesh Kumar K.V * without claiming the space. 1849e56eb659SAneesh Kumar K.V */ 1850e56eb659SAneesh Kumar K.V break; 185126346ff6SAneesh Kumar K.V } 1852c9de560dSAlex Tomas 1853c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 1854c9de560dSAlex Tomas 1855c9de560dSAlex Tomas i += ex.fe_len; 1856c9de560dSAlex Tomas free -= ex.fe_len; 1857c9de560dSAlex Tomas } 1858c9de560dSAlex Tomas 1859c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 1860c9de560dSAlex Tomas } 1861c9de560dSAlex Tomas 1862c9de560dSAlex Tomas /* 1863c9de560dSAlex Tomas * This is a special case for storages like raid5 1864c9de560dSAlex Tomas * we try to find stripe-aligned chunks for stripe-size requests 1865c9de560dSAlex Tomas * XXX should do so at least for multiples of stripe size as well 1866c9de560dSAlex Tomas */ 1867c9de560dSAlex Tomas static void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 1868c9de560dSAlex Tomas struct ext4_buddy *e4b) 1869c9de560dSAlex Tomas { 1870c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1871c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 1872c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1873c9de560dSAlex Tomas struct ext4_free_extent ex; 1874c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 1875c9de560dSAlex Tomas ext4_fsblk_t a; 1876c9de560dSAlex Tomas ext4_grpblk_t i; 1877c9de560dSAlex Tomas int max; 1878c9de560dSAlex Tomas 1879c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 1880c9de560dSAlex Tomas 1881c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 1882c9de560dSAlex Tomas first_group_block = e4b->bd_group * EXT4_BLOCKS_PER_GROUP(sb) 1883c9de560dSAlex Tomas + le32_to_cpu(sbi->s_es->s_first_data_block); 1884c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 1885c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 1886c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 1887c9de560dSAlex Tomas 1888c9de560dSAlex Tomas while (i < EXT4_BLOCKS_PER_GROUP(sb)) { 1889c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 1890c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, i, sbi->s_stripe, &ex); 1891c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 1892c9de560dSAlex Tomas ac->ac_found++; 1893c9de560dSAlex Tomas ac->ac_b_ex = ex; 1894c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1895c9de560dSAlex Tomas break; 1896c9de560dSAlex Tomas } 1897c9de560dSAlex Tomas } 1898c9de560dSAlex Tomas i += sbi->s_stripe; 1899c9de560dSAlex Tomas } 1900c9de560dSAlex Tomas } 1901c9de560dSAlex Tomas 1902c9de560dSAlex Tomas static int ext4_mb_good_group(struct ext4_allocation_context *ac, 1903c9de560dSAlex Tomas ext4_group_t group, int cr) 1904c9de560dSAlex Tomas { 1905c9de560dSAlex Tomas unsigned free, fragments; 1906c9de560dSAlex Tomas unsigned i, bits; 1907c9de560dSAlex Tomas struct ext4_group_desc *desc; 1908c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1909c9de560dSAlex Tomas 1910c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 1911c9de560dSAlex Tomas BUG_ON(EXT4_MB_GRP_NEED_INIT(grp)); 1912c9de560dSAlex Tomas 1913c9de560dSAlex Tomas free = grp->bb_free; 1914c9de560dSAlex Tomas fragments = grp->bb_fragments; 1915c9de560dSAlex Tomas if (free == 0) 1916c9de560dSAlex Tomas return 0; 1917c9de560dSAlex Tomas if (fragments == 0) 1918c9de560dSAlex Tomas return 0; 1919c9de560dSAlex Tomas 1920c9de560dSAlex Tomas switch (cr) { 1921c9de560dSAlex Tomas case 0: 1922c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 1923c9de560dSAlex Tomas /* If this group is uninitialized, skip it initially */ 1924c9de560dSAlex Tomas desc = ext4_get_group_desc(ac->ac_sb, group, NULL); 1925c9de560dSAlex Tomas if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) 1926c9de560dSAlex Tomas return 0; 1927c9de560dSAlex Tomas 1928c9de560dSAlex Tomas bits = ac->ac_sb->s_blocksize_bits + 1; 1929c9de560dSAlex Tomas for (i = ac->ac_2order; i <= bits; i++) 1930c9de560dSAlex Tomas if (grp->bb_counters[i] > 0) 1931c9de560dSAlex Tomas return 1; 1932c9de560dSAlex Tomas break; 1933c9de560dSAlex Tomas case 1: 1934c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 1935c9de560dSAlex Tomas return 1; 1936c9de560dSAlex Tomas break; 1937c9de560dSAlex Tomas case 2: 1938c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 1939c9de560dSAlex Tomas return 1; 1940c9de560dSAlex Tomas break; 1941c9de560dSAlex Tomas case 3: 1942c9de560dSAlex Tomas return 1; 1943c9de560dSAlex Tomas default: 1944c9de560dSAlex Tomas BUG(); 1945c9de560dSAlex Tomas } 1946c9de560dSAlex Tomas 1947c9de560dSAlex Tomas return 0; 1948c9de560dSAlex Tomas } 1949c9de560dSAlex Tomas 1950c9de560dSAlex Tomas static int ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 1951c9de560dSAlex Tomas { 1952c9de560dSAlex Tomas ext4_group_t group; 1953c9de560dSAlex Tomas ext4_group_t i; 1954c9de560dSAlex Tomas int cr; 1955c9de560dSAlex Tomas int err = 0; 1956c9de560dSAlex Tomas int bsbits; 1957c9de560dSAlex Tomas struct ext4_sb_info *sbi; 1958c9de560dSAlex Tomas struct super_block *sb; 1959c9de560dSAlex Tomas struct ext4_buddy e4b; 1960c9de560dSAlex Tomas loff_t size, isize; 1961c9de560dSAlex Tomas 1962c9de560dSAlex Tomas sb = ac->ac_sb; 1963c9de560dSAlex Tomas sbi = EXT4_SB(sb); 1964c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1965c9de560dSAlex Tomas 1966c9de560dSAlex Tomas /* first, try the goal */ 1967c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 1968c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 1969c9de560dSAlex Tomas goto out; 1970c9de560dSAlex Tomas 1971c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 1972c9de560dSAlex Tomas goto out; 1973c9de560dSAlex Tomas 1974c9de560dSAlex Tomas /* 1975c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 1976c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 1977c9de560dSAlex Tomas * try exact allocation using buddy. 1978c9de560dSAlex Tomas */ 1979c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 1980c9de560dSAlex Tomas ac->ac_2order = 0; 1981c9de560dSAlex Tomas /* 1982c9de560dSAlex Tomas * We search using buddy data only if the order of the request 1983c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 1984c9de560dSAlex Tomas * You can tune it via /proc/fs/ext4/<partition>/order2_req 1985c9de560dSAlex Tomas */ 1986c9de560dSAlex Tomas if (i >= sbi->s_mb_order2_reqs) { 1987c9de560dSAlex Tomas /* 1988c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 1989c9de560dSAlex Tomas */ 1990c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 1991c9de560dSAlex Tomas ac->ac_2order = i - 1; 1992c9de560dSAlex Tomas } 1993c9de560dSAlex Tomas 1994c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 1995c9de560dSAlex Tomas /* if stream allocation is enabled, use global goal */ 1996c9de560dSAlex Tomas size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len; 1997c9de560dSAlex Tomas isize = i_size_read(ac->ac_inode) >> bsbits; 1998c9de560dSAlex Tomas if (size < isize) 1999c9de560dSAlex Tomas size = isize; 2000c9de560dSAlex Tomas 2001c9de560dSAlex Tomas if (size < sbi->s_mb_stream_request && 2002c9de560dSAlex Tomas (ac->ac_flags & EXT4_MB_HINT_DATA)) { 2003c9de560dSAlex Tomas /* TBD: may be hot point */ 2004c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2005c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2006c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2007c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2008c9de560dSAlex Tomas } 2009c9de560dSAlex Tomas 2010c9de560dSAlex Tomas /* searching for the right group start from the goal value specified */ 2011c9de560dSAlex Tomas group = ac->ac_g_ex.fe_group; 2012c9de560dSAlex Tomas 2013c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2014c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2015c9de560dSAlex Tomas /* 2016c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2017c9de560dSAlex Tomas * cr == 3 try to get anything 2018c9de560dSAlex Tomas */ 2019c9de560dSAlex Tomas repeat: 2020c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2021c9de560dSAlex Tomas ac->ac_criteria = cr; 2022c9de560dSAlex Tomas for (i = 0; i < EXT4_SB(sb)->s_groups_count; group++, i++) { 2023c9de560dSAlex Tomas struct ext4_group_info *grp; 2024c9de560dSAlex Tomas struct ext4_group_desc *desc; 2025c9de560dSAlex Tomas 2026c9de560dSAlex Tomas if (group == EXT4_SB(sb)->s_groups_count) 2027c9de560dSAlex Tomas group = 0; 2028c9de560dSAlex Tomas 2029c9de560dSAlex Tomas /* quick check to skip empty groups */ 2030c9de560dSAlex Tomas grp = ext4_get_group_info(ac->ac_sb, group); 2031c9de560dSAlex Tomas if (grp->bb_free == 0) 2032c9de560dSAlex Tomas continue; 2033c9de560dSAlex Tomas 2034c9de560dSAlex Tomas /* 2035c9de560dSAlex Tomas * if the group is already init we check whether it is 2036c9de560dSAlex Tomas * a good group and if not we don't load the buddy 2037c9de560dSAlex Tomas */ 2038c9de560dSAlex Tomas if (EXT4_MB_GRP_NEED_INIT(grp)) { 2039c9de560dSAlex Tomas /* 2040c9de560dSAlex Tomas * we need full data about the group 2041c9de560dSAlex Tomas * to make a good selection 2042c9de560dSAlex Tomas */ 2043c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2044c9de560dSAlex Tomas if (err) 2045c9de560dSAlex Tomas goto out; 2046c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2047c9de560dSAlex Tomas } 2048c9de560dSAlex Tomas 2049c9de560dSAlex Tomas /* 2050c9de560dSAlex Tomas * If the particular group doesn't satisfy our 2051c9de560dSAlex Tomas * criteria we continue with the next group 2052c9de560dSAlex Tomas */ 2053c9de560dSAlex Tomas if (!ext4_mb_good_group(ac, group, cr)) 2054c9de560dSAlex Tomas continue; 2055c9de560dSAlex Tomas 2056c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2057c9de560dSAlex Tomas if (err) 2058c9de560dSAlex Tomas goto out; 2059c9de560dSAlex Tomas 2060c9de560dSAlex Tomas ext4_lock_group(sb, group); 2061c9de560dSAlex Tomas if (!ext4_mb_good_group(ac, group, cr)) { 2062c9de560dSAlex Tomas /* someone did allocation from this group */ 2063c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2064c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2065c9de560dSAlex Tomas continue; 2066c9de560dSAlex Tomas } 2067c9de560dSAlex Tomas 2068c9de560dSAlex Tomas ac->ac_groups_scanned++; 2069c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, group, NULL); 2070c9de560dSAlex Tomas if (cr == 0 || (desc->bg_flags & 2071c9de560dSAlex Tomas cpu_to_le16(EXT4_BG_BLOCK_UNINIT) && 2072c9de560dSAlex Tomas ac->ac_2order != 0)) 2073c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2074c9de560dSAlex Tomas else if (cr == 1 && 2075c9de560dSAlex Tomas ac->ac_g_ex.fe_len == sbi->s_stripe) 2076c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2077c9de560dSAlex Tomas else 2078c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2079c9de560dSAlex Tomas 2080c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2081c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2082c9de560dSAlex Tomas 2083c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2084c9de560dSAlex Tomas break; 2085c9de560dSAlex Tomas } 2086c9de560dSAlex Tomas } 2087c9de560dSAlex Tomas 2088c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2089c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2090c9de560dSAlex Tomas /* 2091c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2092c9de560dSAlex Tomas * the best chunk we've found so far 2093c9de560dSAlex Tomas */ 2094c9de560dSAlex Tomas 2095c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2096c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2097c9de560dSAlex Tomas /* 2098c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2099c9de560dSAlex Tomas * The only thing we can do is just take first 2100c9de560dSAlex Tomas * found block(s) 2101c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2102c9de560dSAlex Tomas */ 2103c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2104c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2105c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2106c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2107c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2108c9de560dSAlex Tomas cr = 3; 2109c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2110c9de560dSAlex Tomas goto repeat; 2111c9de560dSAlex Tomas } 2112c9de560dSAlex Tomas } 2113c9de560dSAlex Tomas out: 2114c9de560dSAlex Tomas return err; 2115c9de560dSAlex Tomas } 2116c9de560dSAlex Tomas 2117c9de560dSAlex Tomas #ifdef EXT4_MB_HISTORY 2118c9de560dSAlex Tomas struct ext4_mb_proc_session { 2119c9de560dSAlex Tomas struct ext4_mb_history *history; 2120c9de560dSAlex Tomas struct super_block *sb; 2121c9de560dSAlex Tomas int start; 2122c9de560dSAlex Tomas int max; 2123c9de560dSAlex Tomas }; 2124c9de560dSAlex Tomas 2125c9de560dSAlex Tomas static void *ext4_mb_history_skip_empty(struct ext4_mb_proc_session *s, 2126c9de560dSAlex Tomas struct ext4_mb_history *hs, 2127c9de560dSAlex Tomas int first) 2128c9de560dSAlex Tomas { 2129c9de560dSAlex Tomas if (hs == s->history + s->max) 2130c9de560dSAlex Tomas hs = s->history; 2131c9de560dSAlex Tomas if (!first && hs == s->history + s->start) 2132c9de560dSAlex Tomas return NULL; 2133c9de560dSAlex Tomas while (hs->orig.fe_len == 0) { 2134c9de560dSAlex Tomas hs++; 2135c9de560dSAlex Tomas if (hs == s->history + s->max) 2136c9de560dSAlex Tomas hs = s->history; 2137c9de560dSAlex Tomas if (hs == s->history + s->start) 2138c9de560dSAlex Tomas return NULL; 2139c9de560dSAlex Tomas } 2140c9de560dSAlex Tomas return hs; 2141c9de560dSAlex Tomas } 2142c9de560dSAlex Tomas 2143c9de560dSAlex Tomas static void *ext4_mb_seq_history_start(struct seq_file *seq, loff_t *pos) 2144c9de560dSAlex Tomas { 2145c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2146c9de560dSAlex Tomas struct ext4_mb_history *hs; 2147c9de560dSAlex Tomas int l = *pos; 2148c9de560dSAlex Tomas 2149c9de560dSAlex Tomas if (l == 0) 2150c9de560dSAlex Tomas return SEQ_START_TOKEN; 2151c9de560dSAlex Tomas hs = ext4_mb_history_skip_empty(s, s->history + s->start, 1); 2152c9de560dSAlex Tomas if (!hs) 2153c9de560dSAlex Tomas return NULL; 2154c9de560dSAlex Tomas while (--l && (hs = ext4_mb_history_skip_empty(s, ++hs, 0)) != NULL); 2155c9de560dSAlex Tomas return hs; 2156c9de560dSAlex Tomas } 2157c9de560dSAlex Tomas 2158c9de560dSAlex Tomas static void *ext4_mb_seq_history_next(struct seq_file *seq, void *v, 2159c9de560dSAlex Tomas loff_t *pos) 2160c9de560dSAlex Tomas { 2161c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2162c9de560dSAlex Tomas struct ext4_mb_history *hs = v; 2163c9de560dSAlex Tomas 2164c9de560dSAlex Tomas ++*pos; 2165c9de560dSAlex Tomas if (v == SEQ_START_TOKEN) 2166c9de560dSAlex Tomas return ext4_mb_history_skip_empty(s, s->history + s->start, 1); 2167c9de560dSAlex Tomas else 2168c9de560dSAlex Tomas return ext4_mb_history_skip_empty(s, ++hs, 0); 2169c9de560dSAlex Tomas } 2170c9de560dSAlex Tomas 2171c9de560dSAlex Tomas static int ext4_mb_seq_history_show(struct seq_file *seq, void *v) 2172c9de560dSAlex Tomas { 2173c9de560dSAlex Tomas char buf[25], buf2[25], buf3[25], *fmt; 2174c9de560dSAlex Tomas struct ext4_mb_history *hs = v; 2175c9de560dSAlex Tomas 2176c9de560dSAlex Tomas if (v == SEQ_START_TOKEN) { 2177c9de560dSAlex Tomas seq_printf(seq, "%-5s %-8s %-23s %-23s %-23s %-5s " 2178c9de560dSAlex Tomas "%-5s %-2s %-5s %-5s %-5s %-6s\n", 2179c9de560dSAlex Tomas "pid", "inode", "original", "goal", "result", "found", 2180c9de560dSAlex Tomas "grps", "cr", "flags", "merge", "tail", "broken"); 2181c9de560dSAlex Tomas return 0; 2182c9de560dSAlex Tomas } 2183c9de560dSAlex Tomas 2184c9de560dSAlex Tomas if (hs->op == EXT4_MB_HISTORY_ALLOC) { 2185c9de560dSAlex Tomas fmt = "%-5u %-8u %-23s %-23s %-23s %-5u %-5u %-2u " 2186c9de560dSAlex Tomas "%-5u %-5s %-5u %-6u\n"; 2187c9de560dSAlex Tomas sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group, 2188c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len, 2189c9de560dSAlex Tomas hs->result.fe_logical); 2190c9de560dSAlex Tomas sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group, 2191c9de560dSAlex Tomas hs->orig.fe_start, hs->orig.fe_len, 2192c9de560dSAlex Tomas hs->orig.fe_logical); 2193c9de560dSAlex Tomas sprintf(buf3, "%lu/%d/%u@%u", hs->goal.fe_group, 2194c9de560dSAlex Tomas hs->goal.fe_start, hs->goal.fe_len, 2195c9de560dSAlex Tomas hs->goal.fe_logical); 2196c9de560dSAlex Tomas seq_printf(seq, fmt, hs->pid, hs->ino, buf, buf3, buf2, 2197c9de560dSAlex Tomas hs->found, hs->groups, hs->cr, hs->flags, 2198c9de560dSAlex Tomas hs->merged ? "M" : "", hs->tail, 2199c9de560dSAlex Tomas hs->buddy ? 1 << hs->buddy : 0); 2200c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_PREALLOC) { 2201c9de560dSAlex Tomas fmt = "%-5u %-8u %-23s %-23s %-23s\n"; 2202c9de560dSAlex Tomas sprintf(buf2, "%lu/%d/%u@%u", hs->result.fe_group, 2203c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len, 2204c9de560dSAlex Tomas hs->result.fe_logical); 2205c9de560dSAlex Tomas sprintf(buf, "%lu/%d/%u@%u", hs->orig.fe_group, 2206c9de560dSAlex Tomas hs->orig.fe_start, hs->orig.fe_len, 2207c9de560dSAlex Tomas hs->orig.fe_logical); 2208c9de560dSAlex Tomas seq_printf(seq, fmt, hs->pid, hs->ino, buf, "", buf2); 2209c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_DISCARD) { 2210c9de560dSAlex Tomas sprintf(buf2, "%lu/%d/%u", hs->result.fe_group, 2211c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len); 2212c9de560dSAlex Tomas seq_printf(seq, "%-5u %-8u %-23s discard\n", 2213c9de560dSAlex Tomas hs->pid, hs->ino, buf2); 2214c9de560dSAlex Tomas } else if (hs->op == EXT4_MB_HISTORY_FREE) { 2215c9de560dSAlex Tomas sprintf(buf2, "%lu/%d/%u", hs->result.fe_group, 2216c9de560dSAlex Tomas hs->result.fe_start, hs->result.fe_len); 2217c9de560dSAlex Tomas seq_printf(seq, "%-5u %-8u %-23s free\n", 2218c9de560dSAlex Tomas hs->pid, hs->ino, buf2); 2219c9de560dSAlex Tomas } 2220c9de560dSAlex Tomas return 0; 2221c9de560dSAlex Tomas } 2222c9de560dSAlex Tomas 2223c9de560dSAlex Tomas static void ext4_mb_seq_history_stop(struct seq_file *seq, void *v) 2224c9de560dSAlex Tomas { 2225c9de560dSAlex Tomas } 2226c9de560dSAlex Tomas 2227c9de560dSAlex Tomas static struct seq_operations ext4_mb_seq_history_ops = { 2228c9de560dSAlex Tomas .start = ext4_mb_seq_history_start, 2229c9de560dSAlex Tomas .next = ext4_mb_seq_history_next, 2230c9de560dSAlex Tomas .stop = ext4_mb_seq_history_stop, 2231c9de560dSAlex Tomas .show = ext4_mb_seq_history_show, 2232c9de560dSAlex Tomas }; 2233c9de560dSAlex Tomas 2234c9de560dSAlex Tomas static int ext4_mb_seq_history_open(struct inode *inode, struct file *file) 2235c9de560dSAlex Tomas { 2236c9de560dSAlex Tomas struct super_block *sb = PDE(inode)->data; 2237c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2238c9de560dSAlex Tomas struct ext4_mb_proc_session *s; 2239c9de560dSAlex Tomas int rc; 2240c9de560dSAlex Tomas int size; 2241c9de560dSAlex Tomas 2242c9de560dSAlex Tomas s = kmalloc(sizeof(*s), GFP_KERNEL); 2243c9de560dSAlex Tomas if (s == NULL) 2244c9de560dSAlex Tomas return -ENOMEM; 2245c9de560dSAlex Tomas s->sb = sb; 2246c9de560dSAlex Tomas size = sizeof(struct ext4_mb_history) * sbi->s_mb_history_max; 2247c9de560dSAlex Tomas s->history = kmalloc(size, GFP_KERNEL); 2248c9de560dSAlex Tomas if (s->history == NULL) { 2249c9de560dSAlex Tomas kfree(s); 2250c9de560dSAlex Tomas return -ENOMEM; 2251c9de560dSAlex Tomas } 2252c9de560dSAlex Tomas 2253c9de560dSAlex Tomas spin_lock(&sbi->s_mb_history_lock); 2254c9de560dSAlex Tomas memcpy(s->history, sbi->s_mb_history, size); 2255c9de560dSAlex Tomas s->max = sbi->s_mb_history_max; 2256c9de560dSAlex Tomas s->start = sbi->s_mb_history_cur % s->max; 2257c9de560dSAlex Tomas spin_unlock(&sbi->s_mb_history_lock); 2258c9de560dSAlex Tomas 2259c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_history_ops); 2260c9de560dSAlex Tomas if (rc == 0) { 2261c9de560dSAlex Tomas struct seq_file *m = (struct seq_file *)file->private_data; 2262c9de560dSAlex Tomas m->private = s; 2263c9de560dSAlex Tomas } else { 2264c9de560dSAlex Tomas kfree(s->history); 2265c9de560dSAlex Tomas kfree(s); 2266c9de560dSAlex Tomas } 2267c9de560dSAlex Tomas return rc; 2268c9de560dSAlex Tomas 2269c9de560dSAlex Tomas } 2270c9de560dSAlex Tomas 2271c9de560dSAlex Tomas static int ext4_mb_seq_history_release(struct inode *inode, struct file *file) 2272c9de560dSAlex Tomas { 2273c9de560dSAlex Tomas struct seq_file *seq = (struct seq_file *)file->private_data; 2274c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2275c9de560dSAlex Tomas kfree(s->history); 2276c9de560dSAlex Tomas kfree(s); 2277c9de560dSAlex Tomas return seq_release(inode, file); 2278c9de560dSAlex Tomas } 2279c9de560dSAlex Tomas 2280c9de560dSAlex Tomas static ssize_t ext4_mb_seq_history_write(struct file *file, 2281c9de560dSAlex Tomas const char __user *buffer, 2282c9de560dSAlex Tomas size_t count, loff_t *ppos) 2283c9de560dSAlex Tomas { 2284c9de560dSAlex Tomas struct seq_file *seq = (struct seq_file *)file->private_data; 2285c9de560dSAlex Tomas struct ext4_mb_proc_session *s = seq->private; 2286c9de560dSAlex Tomas struct super_block *sb = s->sb; 2287c9de560dSAlex Tomas char str[32]; 2288c9de560dSAlex Tomas int value; 2289c9de560dSAlex Tomas 2290c9de560dSAlex Tomas if (count >= sizeof(str)) { 2291c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: %s string too long, max %u bytes\n", 2292c9de560dSAlex Tomas "mb_history", (int)sizeof(str)); 2293c9de560dSAlex Tomas return -EOVERFLOW; 2294c9de560dSAlex Tomas } 2295c9de560dSAlex Tomas 2296c9de560dSAlex Tomas if (copy_from_user(str, buffer, count)) 2297c9de560dSAlex Tomas return -EFAULT; 2298c9de560dSAlex Tomas 2299c9de560dSAlex Tomas value = simple_strtol(str, NULL, 0); 2300c9de560dSAlex Tomas if (value < 0) 2301c9de560dSAlex Tomas return -ERANGE; 2302c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_history_filter = value; 2303c9de560dSAlex Tomas 2304c9de560dSAlex Tomas return count; 2305c9de560dSAlex Tomas } 2306c9de560dSAlex Tomas 2307c9de560dSAlex Tomas static struct file_operations ext4_mb_seq_history_fops = { 2308c9de560dSAlex Tomas .owner = THIS_MODULE, 2309c9de560dSAlex Tomas .open = ext4_mb_seq_history_open, 2310c9de560dSAlex Tomas .read = seq_read, 2311c9de560dSAlex Tomas .write = ext4_mb_seq_history_write, 2312c9de560dSAlex Tomas .llseek = seq_lseek, 2313c9de560dSAlex Tomas .release = ext4_mb_seq_history_release, 2314c9de560dSAlex Tomas }; 2315c9de560dSAlex Tomas 2316c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2317c9de560dSAlex Tomas { 2318c9de560dSAlex Tomas struct super_block *sb = seq->private; 2319c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2320c9de560dSAlex Tomas ext4_group_t group; 2321c9de560dSAlex Tomas 2322c9de560dSAlex Tomas if (*pos < 0 || *pos >= sbi->s_groups_count) 2323c9de560dSAlex Tomas return NULL; 2324c9de560dSAlex Tomas 2325c9de560dSAlex Tomas group = *pos + 1; 2326c9de560dSAlex Tomas return (void *) group; 2327c9de560dSAlex Tomas } 2328c9de560dSAlex Tomas 2329c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2330c9de560dSAlex Tomas { 2331c9de560dSAlex Tomas struct super_block *sb = seq->private; 2332c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2333c9de560dSAlex Tomas ext4_group_t group; 2334c9de560dSAlex Tomas 2335c9de560dSAlex Tomas ++*pos; 2336c9de560dSAlex Tomas if (*pos < 0 || *pos >= sbi->s_groups_count) 2337c9de560dSAlex Tomas return NULL; 2338c9de560dSAlex Tomas group = *pos + 1; 2339c9de560dSAlex Tomas return (void *) group;; 2340c9de560dSAlex Tomas } 2341c9de560dSAlex Tomas 2342c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2343c9de560dSAlex Tomas { 2344c9de560dSAlex Tomas struct super_block *sb = seq->private; 2345c9de560dSAlex Tomas long group = (long) v; 2346c9de560dSAlex Tomas int i; 2347c9de560dSAlex Tomas int err; 2348c9de560dSAlex Tomas struct ext4_buddy e4b; 2349c9de560dSAlex Tomas struct sg { 2350c9de560dSAlex Tomas struct ext4_group_info info; 2351c9de560dSAlex Tomas unsigned short counters[16]; 2352c9de560dSAlex Tomas } sg; 2353c9de560dSAlex Tomas 2354c9de560dSAlex Tomas group--; 2355c9de560dSAlex Tomas if (group == 0) 2356c9de560dSAlex Tomas seq_printf(seq, "#%-5s: %-5s %-5s %-5s " 2357c9de560dSAlex Tomas "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s " 2358c9de560dSAlex Tomas "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n", 2359c9de560dSAlex Tomas "group", "free", "frags", "first", 2360c9de560dSAlex Tomas "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6", 2361c9de560dSAlex Tomas "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13"); 2362c9de560dSAlex Tomas 2363c9de560dSAlex Tomas i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2364c9de560dSAlex Tomas sizeof(struct ext4_group_info); 2365c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2366c9de560dSAlex Tomas if (err) { 2367c9de560dSAlex Tomas seq_printf(seq, "#%-5lu: I/O error\n", group); 2368c9de560dSAlex Tomas return 0; 2369c9de560dSAlex Tomas } 2370c9de560dSAlex Tomas ext4_lock_group(sb, group); 2371c9de560dSAlex Tomas memcpy(&sg, ext4_get_group_info(sb, group), i); 2372c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2373c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2374c9de560dSAlex Tomas 2375c9de560dSAlex Tomas seq_printf(seq, "#%-5lu: %-5u %-5u %-5u [", group, sg.info.bb_free, 2376c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2377c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 2378c9de560dSAlex Tomas seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ? 2379c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2380c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2381c9de560dSAlex Tomas 2382c9de560dSAlex Tomas return 0; 2383c9de560dSAlex Tomas } 2384c9de560dSAlex Tomas 2385c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2386c9de560dSAlex Tomas { 2387c9de560dSAlex Tomas } 2388c9de560dSAlex Tomas 2389c9de560dSAlex Tomas static struct seq_operations ext4_mb_seq_groups_ops = { 2390c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2391c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2392c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2393c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2394c9de560dSAlex Tomas }; 2395c9de560dSAlex Tomas 2396c9de560dSAlex Tomas static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file) 2397c9de560dSAlex Tomas { 2398c9de560dSAlex Tomas struct super_block *sb = PDE(inode)->data; 2399c9de560dSAlex Tomas int rc; 2400c9de560dSAlex Tomas 2401c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_groups_ops); 2402c9de560dSAlex Tomas if (rc == 0) { 2403c9de560dSAlex Tomas struct seq_file *m = (struct seq_file *)file->private_data; 2404c9de560dSAlex Tomas m->private = sb; 2405c9de560dSAlex Tomas } 2406c9de560dSAlex Tomas return rc; 2407c9de560dSAlex Tomas 2408c9de560dSAlex Tomas } 2409c9de560dSAlex Tomas 2410c9de560dSAlex Tomas static struct file_operations ext4_mb_seq_groups_fops = { 2411c9de560dSAlex Tomas .owner = THIS_MODULE, 2412c9de560dSAlex Tomas .open = ext4_mb_seq_groups_open, 2413c9de560dSAlex Tomas .read = seq_read, 2414c9de560dSAlex Tomas .llseek = seq_lseek, 2415c9de560dSAlex Tomas .release = seq_release, 2416c9de560dSAlex Tomas }; 2417c9de560dSAlex Tomas 2418c9de560dSAlex Tomas static void ext4_mb_history_release(struct super_block *sb) 2419c9de560dSAlex Tomas { 2420c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2421c9de560dSAlex Tomas 2422c9de560dSAlex Tomas remove_proc_entry("mb_groups", sbi->s_mb_proc); 2423c9de560dSAlex Tomas remove_proc_entry("mb_history", sbi->s_mb_proc); 2424c9de560dSAlex Tomas 2425c9de560dSAlex Tomas kfree(sbi->s_mb_history); 2426c9de560dSAlex Tomas } 2427c9de560dSAlex Tomas 2428c9de560dSAlex Tomas static void ext4_mb_history_init(struct super_block *sb) 2429c9de560dSAlex Tomas { 2430c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2431c9de560dSAlex Tomas int i; 2432c9de560dSAlex Tomas 2433c9de560dSAlex Tomas if (sbi->s_mb_proc != NULL) { 2434c9de560dSAlex Tomas struct proc_dir_entry *p; 2435c9de560dSAlex Tomas p = create_proc_entry("mb_history", S_IRUGO, sbi->s_mb_proc); 2436c9de560dSAlex Tomas if (p) { 2437c9de560dSAlex Tomas p->proc_fops = &ext4_mb_seq_history_fops; 2438c9de560dSAlex Tomas p->data = sb; 2439c9de560dSAlex Tomas } 2440c9de560dSAlex Tomas p = create_proc_entry("mb_groups", S_IRUGO, sbi->s_mb_proc); 2441c9de560dSAlex Tomas if (p) { 2442c9de560dSAlex Tomas p->proc_fops = &ext4_mb_seq_groups_fops; 2443c9de560dSAlex Tomas p->data = sb; 2444c9de560dSAlex Tomas } 2445c9de560dSAlex Tomas } 2446c9de560dSAlex Tomas 2447c9de560dSAlex Tomas sbi->s_mb_history_max = 1000; 2448c9de560dSAlex Tomas sbi->s_mb_history_cur = 0; 2449c9de560dSAlex Tomas spin_lock_init(&sbi->s_mb_history_lock); 2450c9de560dSAlex Tomas i = sbi->s_mb_history_max * sizeof(struct ext4_mb_history); 2451c9de560dSAlex Tomas sbi->s_mb_history = kmalloc(i, GFP_KERNEL); 2452c9de560dSAlex Tomas if (likely(sbi->s_mb_history != NULL)) 2453c9de560dSAlex Tomas memset(sbi->s_mb_history, 0, i); 2454c9de560dSAlex Tomas /* if we can't allocate history, then we simple won't use it */ 2455c9de560dSAlex Tomas } 2456c9de560dSAlex Tomas 2457c9de560dSAlex Tomas static void ext4_mb_store_history(struct ext4_allocation_context *ac) 2458c9de560dSAlex Tomas { 2459c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2460c9de560dSAlex Tomas struct ext4_mb_history h; 2461c9de560dSAlex Tomas 2462c9de560dSAlex Tomas if (unlikely(sbi->s_mb_history == NULL)) 2463c9de560dSAlex Tomas return; 2464c9de560dSAlex Tomas 2465c9de560dSAlex Tomas if (!(ac->ac_op & sbi->s_mb_history_filter)) 2466c9de560dSAlex Tomas return; 2467c9de560dSAlex Tomas 2468c9de560dSAlex Tomas h.op = ac->ac_op; 2469c9de560dSAlex Tomas h.pid = current->pid; 2470c9de560dSAlex Tomas h.ino = ac->ac_inode ? ac->ac_inode->i_ino : 0; 2471c9de560dSAlex Tomas h.orig = ac->ac_o_ex; 2472c9de560dSAlex Tomas h.result = ac->ac_b_ex; 2473c9de560dSAlex Tomas h.flags = ac->ac_flags; 2474c9de560dSAlex Tomas h.found = ac->ac_found; 2475c9de560dSAlex Tomas h.groups = ac->ac_groups_scanned; 2476c9de560dSAlex Tomas h.cr = ac->ac_criteria; 2477c9de560dSAlex Tomas h.tail = ac->ac_tail; 2478c9de560dSAlex Tomas h.buddy = ac->ac_buddy; 2479c9de560dSAlex Tomas h.merged = 0; 2480c9de560dSAlex Tomas if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) { 2481c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 2482c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 2483c9de560dSAlex Tomas h.merged = 1; 2484c9de560dSAlex Tomas h.goal = ac->ac_g_ex; 2485c9de560dSAlex Tomas h.result = ac->ac_f_ex; 2486c9de560dSAlex Tomas } 2487c9de560dSAlex Tomas 2488c9de560dSAlex Tomas spin_lock(&sbi->s_mb_history_lock); 2489c9de560dSAlex Tomas memcpy(sbi->s_mb_history + sbi->s_mb_history_cur, &h, sizeof(h)); 2490c9de560dSAlex Tomas if (++sbi->s_mb_history_cur >= sbi->s_mb_history_max) 2491c9de560dSAlex Tomas sbi->s_mb_history_cur = 0; 2492c9de560dSAlex Tomas spin_unlock(&sbi->s_mb_history_lock); 2493c9de560dSAlex Tomas } 2494c9de560dSAlex Tomas 2495c9de560dSAlex Tomas #else 2496c9de560dSAlex Tomas #define ext4_mb_history_release(sb) 2497c9de560dSAlex Tomas #define ext4_mb_history_init(sb) 2498c9de560dSAlex Tomas #endif 2499c9de560dSAlex Tomas 2500c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2501c9de560dSAlex Tomas { 2502c9de560dSAlex Tomas ext4_group_t i; 2503c9de560dSAlex Tomas int j, len, metalen; 2504c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2505c9de560dSAlex Tomas int num_meta_group_infos = 2506c9de560dSAlex Tomas (sbi->s_groups_count + EXT4_DESC_PER_BLOCK(sb) - 1) >> 2507c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2508c9de560dSAlex Tomas struct ext4_group_info **meta_group_info; 2509c9de560dSAlex Tomas 2510c9de560dSAlex Tomas /* An 8TB filesystem with 64-bit pointers requires a 4096 byte 2511c9de560dSAlex Tomas * kmalloc. A 128kb malloc should suffice for a 256TB filesystem. 2512c9de560dSAlex Tomas * So a two level scheme suffices for now. */ 2513c9de560dSAlex Tomas sbi->s_group_info = kmalloc(sizeof(*sbi->s_group_info) * 2514c9de560dSAlex Tomas num_meta_group_infos, GFP_KERNEL); 2515c9de560dSAlex Tomas if (sbi->s_group_info == NULL) { 2516c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't allocate buddy meta group\n"); 2517c9de560dSAlex Tomas return -ENOMEM; 2518c9de560dSAlex Tomas } 2519c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2520c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 2521c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't get new inode\n"); 2522c9de560dSAlex Tomas goto err_freesgi; 2523c9de560dSAlex Tomas } 2524c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 2525c9de560dSAlex Tomas 2526c9de560dSAlex Tomas metalen = sizeof(*meta_group_info) << EXT4_DESC_PER_BLOCK_BITS(sb); 2527c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) { 2528c9de560dSAlex Tomas if ((i + 1) == num_meta_group_infos) 2529c9de560dSAlex Tomas metalen = sizeof(*meta_group_info) * 2530c9de560dSAlex Tomas (sbi->s_groups_count - 2531c9de560dSAlex Tomas (i << EXT4_DESC_PER_BLOCK_BITS(sb))); 2532c9de560dSAlex Tomas meta_group_info = kmalloc(metalen, GFP_KERNEL); 2533c9de560dSAlex Tomas if (meta_group_info == NULL) { 2534c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't allocate mem for a " 2535c9de560dSAlex Tomas "buddy group\n"); 2536c9de560dSAlex Tomas goto err_freemeta; 2537c9de560dSAlex Tomas } 2538c9de560dSAlex Tomas sbi->s_group_info[i] = meta_group_info; 2539c9de560dSAlex Tomas } 2540c9de560dSAlex Tomas 2541c9de560dSAlex Tomas /* 2542c9de560dSAlex Tomas * calculate needed size. if change bb_counters size, 2543c9de560dSAlex Tomas * don't forget about ext4_mb_generate_buddy() 2544c9de560dSAlex Tomas */ 2545c9de560dSAlex Tomas len = sizeof(struct ext4_group_info); 2546c9de560dSAlex Tomas len += sizeof(unsigned short) * (sb->s_blocksize_bits + 2); 2547c9de560dSAlex Tomas for (i = 0; i < sbi->s_groups_count; i++) { 2548c9de560dSAlex Tomas struct ext4_group_desc *desc; 2549c9de560dSAlex Tomas 2550c9de560dSAlex Tomas meta_group_info = 2551c9de560dSAlex Tomas sbi->s_group_info[i >> EXT4_DESC_PER_BLOCK_BITS(sb)]; 2552c9de560dSAlex Tomas j = i & (EXT4_DESC_PER_BLOCK(sb) - 1); 2553c9de560dSAlex Tomas 2554c9de560dSAlex Tomas meta_group_info[j] = kzalloc(len, GFP_KERNEL); 2555c9de560dSAlex Tomas if (meta_group_info[j] == NULL) { 2556c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't allocate buddy mem\n"); 2557c9de560dSAlex Tomas i--; 2558c9de560dSAlex Tomas goto err_freebuddy; 2559c9de560dSAlex Tomas } 2560c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2561c9de560dSAlex Tomas if (desc == NULL) { 2562c9de560dSAlex Tomas printk(KERN_ERR 2563c9de560dSAlex Tomas "EXT4-fs: can't read descriptor %lu\n", i); 2564c9de560dSAlex Tomas goto err_freebuddy; 2565c9de560dSAlex Tomas } 2566c9de560dSAlex Tomas memset(meta_group_info[j], 0, len); 2567c9de560dSAlex Tomas set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 2568c9de560dSAlex Tomas &(meta_group_info[j]->bb_state)); 2569c9de560dSAlex Tomas 2570c9de560dSAlex Tomas /* 2571c9de560dSAlex Tomas * initialize bb_free to be able to skip 2572c9de560dSAlex Tomas * empty groups without initialization 2573c9de560dSAlex Tomas */ 2574c9de560dSAlex Tomas if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 2575c9de560dSAlex Tomas meta_group_info[j]->bb_free = 2576c9de560dSAlex Tomas ext4_free_blocks_after_init(sb, i, desc); 2577c9de560dSAlex Tomas } else { 2578c9de560dSAlex Tomas meta_group_info[j]->bb_free = 2579c9de560dSAlex Tomas le16_to_cpu(desc->bg_free_blocks_count); 2580c9de560dSAlex Tomas } 2581c9de560dSAlex Tomas 2582c9de560dSAlex Tomas INIT_LIST_HEAD(&meta_group_info[j]->bb_prealloc_list); 2583c9de560dSAlex Tomas 2584c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2585c9de560dSAlex Tomas { 2586c9de560dSAlex Tomas struct buffer_head *bh; 2587c9de560dSAlex Tomas meta_group_info[j]->bb_bitmap = 2588c9de560dSAlex Tomas kmalloc(sb->s_blocksize, GFP_KERNEL); 2589c9de560dSAlex Tomas BUG_ON(meta_group_info[j]->bb_bitmap == NULL); 2590c9de560dSAlex Tomas bh = read_block_bitmap(sb, i); 2591c9de560dSAlex Tomas BUG_ON(bh == NULL); 2592c9de560dSAlex Tomas memcpy(meta_group_info[j]->bb_bitmap, bh->b_data, 2593c9de560dSAlex Tomas sb->s_blocksize); 2594c9de560dSAlex Tomas put_bh(bh); 2595c9de560dSAlex Tomas } 2596c9de560dSAlex Tomas #endif 2597c9de560dSAlex Tomas 2598c9de560dSAlex Tomas } 2599c9de560dSAlex Tomas 2600c9de560dSAlex Tomas return 0; 2601c9de560dSAlex Tomas 2602c9de560dSAlex Tomas err_freebuddy: 2603c9de560dSAlex Tomas while (i >= 0) { 2604c9de560dSAlex Tomas kfree(ext4_get_group_info(sb, i)); 2605c9de560dSAlex Tomas i--; 2606c9de560dSAlex Tomas } 2607c9de560dSAlex Tomas i = num_meta_group_infos; 2608c9de560dSAlex Tomas err_freemeta: 2609c9de560dSAlex Tomas while (--i >= 0) 2610c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2611c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2612c9de560dSAlex Tomas err_freesgi: 2613c9de560dSAlex Tomas kfree(sbi->s_group_info); 2614c9de560dSAlex Tomas return -ENOMEM; 2615c9de560dSAlex Tomas } 2616c9de560dSAlex Tomas 2617c9de560dSAlex Tomas int ext4_mb_init(struct super_block *sb, int needs_recovery) 2618c9de560dSAlex Tomas { 2619c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2620c9de560dSAlex Tomas unsigned i; 2621c9de560dSAlex Tomas unsigned offset; 2622c9de560dSAlex Tomas unsigned max; 2623c9de560dSAlex Tomas 2624c9de560dSAlex Tomas if (!test_opt(sb, MBALLOC)) 2625c9de560dSAlex Tomas return 0; 2626c9de560dSAlex Tomas 2627c9de560dSAlex Tomas i = (sb->s_blocksize_bits + 2) * sizeof(unsigned short); 2628c9de560dSAlex Tomas 2629c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2630c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2631c9de560dSAlex Tomas clear_opt(sbi->s_mount_opt, MBALLOC); 2632c9de560dSAlex Tomas return -ENOMEM; 2633c9de560dSAlex Tomas } 2634c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2635c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2636c9de560dSAlex Tomas clear_opt(sbi->s_mount_opt, MBALLOC); 2637c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2638c9de560dSAlex Tomas return -ENOMEM; 2639c9de560dSAlex Tomas } 2640c9de560dSAlex Tomas 2641c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2642c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2643c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2644c9de560dSAlex Tomas 2645c9de560dSAlex Tomas i = 1; 2646c9de560dSAlex Tomas offset = 0; 2647c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2648c9de560dSAlex Tomas do { 2649c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2650c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2651c9de560dSAlex Tomas offset += 1 << (sb->s_blocksize_bits - i); 2652c9de560dSAlex Tomas max = max >> 1; 2653c9de560dSAlex Tomas i++; 2654c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2655c9de560dSAlex Tomas 2656c9de560dSAlex Tomas /* init file for buddy data */ 2657c9de560dSAlex Tomas i = ext4_mb_init_backend(sb); 2658c9de560dSAlex Tomas if (i) { 2659c9de560dSAlex Tomas clear_opt(sbi->s_mount_opt, MBALLOC); 2660c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2661c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2662c9de560dSAlex Tomas return i; 2663c9de560dSAlex Tomas } 2664c9de560dSAlex Tomas 2665c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2666c9de560dSAlex Tomas INIT_LIST_HEAD(&sbi->s_active_transaction); 2667c9de560dSAlex Tomas INIT_LIST_HEAD(&sbi->s_closed_transaction); 2668c9de560dSAlex Tomas INIT_LIST_HEAD(&sbi->s_committed_transaction); 2669c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2670c9de560dSAlex Tomas 2671c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2672c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2673c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2674c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2675c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 2676c9de560dSAlex Tomas sbi->s_mb_history_filter = EXT4_MB_HISTORY_DEFAULT; 2677c9de560dSAlex Tomas sbi->s_mb_group_prealloc = MB_DEFAULT_GROUP_PREALLOC; 2678c9de560dSAlex Tomas 2679c9de560dSAlex Tomas i = sizeof(struct ext4_locality_group) * NR_CPUS; 2680c9de560dSAlex Tomas sbi->s_locality_groups = kmalloc(i, GFP_KERNEL); 2681c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2682c9de560dSAlex Tomas clear_opt(sbi->s_mount_opt, MBALLOC); 2683c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2684c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2685c9de560dSAlex Tomas return -ENOMEM; 2686c9de560dSAlex Tomas } 2687c9de560dSAlex Tomas for (i = 0; i < NR_CPUS; i++) { 2688c9de560dSAlex Tomas struct ext4_locality_group *lg; 2689c9de560dSAlex Tomas lg = &sbi->s_locality_groups[i]; 2690c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 2691c9de560dSAlex Tomas INIT_LIST_HEAD(&lg->lg_prealloc_list); 2692c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2693c9de560dSAlex Tomas } 2694c9de560dSAlex Tomas 2695c9de560dSAlex Tomas ext4_mb_init_per_dev_proc(sb); 2696c9de560dSAlex Tomas ext4_mb_history_init(sb); 2697c9de560dSAlex Tomas 2698c9de560dSAlex Tomas printk("EXT4-fs: mballoc enabled\n"); 2699c9de560dSAlex Tomas return 0; 2700c9de560dSAlex Tomas } 2701c9de560dSAlex Tomas 2702c9de560dSAlex Tomas /* need to called with ext4 group lock (ext4_lock_group) */ 2703c9de560dSAlex Tomas static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2704c9de560dSAlex Tomas { 2705c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2706c9de560dSAlex Tomas struct list_head *cur, *tmp; 2707c9de560dSAlex Tomas int count = 0; 2708c9de560dSAlex Tomas 2709c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2710c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2711c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2712c9de560dSAlex Tomas count++; 2713c9de560dSAlex Tomas kfree(pa); 2714c9de560dSAlex Tomas } 2715c9de560dSAlex Tomas if (count) 2716c9de560dSAlex Tomas mb_debug("mballoc: %u PAs left\n", count); 2717c9de560dSAlex Tomas 2718c9de560dSAlex Tomas } 2719c9de560dSAlex Tomas 2720c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2721c9de560dSAlex Tomas { 2722c9de560dSAlex Tomas ext4_group_t i; 2723c9de560dSAlex Tomas int num_meta_group_infos; 2724c9de560dSAlex Tomas struct ext4_group_info *grinfo; 2725c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2726c9de560dSAlex Tomas 2727c9de560dSAlex Tomas if (!test_opt(sb, MBALLOC)) 2728c9de560dSAlex Tomas return 0; 2729c9de560dSAlex Tomas 2730c9de560dSAlex Tomas /* release freed, non-committed blocks */ 2731c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2732c9de560dSAlex Tomas list_splice_init(&sbi->s_closed_transaction, 2733c9de560dSAlex Tomas &sbi->s_committed_transaction); 2734c9de560dSAlex Tomas list_splice_init(&sbi->s_active_transaction, 2735c9de560dSAlex Tomas &sbi->s_committed_transaction); 2736c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2737c9de560dSAlex Tomas ext4_mb_free_committed_blocks(sb); 2738c9de560dSAlex Tomas 2739c9de560dSAlex Tomas if (sbi->s_group_info) { 2740c9de560dSAlex Tomas for (i = 0; i < sbi->s_groups_count; i++) { 2741c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2742c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2743c9de560dSAlex Tomas kfree(grinfo->bb_bitmap); 2744c9de560dSAlex Tomas #endif 2745c9de560dSAlex Tomas ext4_lock_group(sb, i); 2746c9de560dSAlex Tomas ext4_mb_cleanup_pa(grinfo); 2747c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2748c9de560dSAlex Tomas kfree(grinfo); 2749c9de560dSAlex Tomas } 2750c9de560dSAlex Tomas num_meta_group_infos = (sbi->s_groups_count + 2751c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2752c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2753c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2754c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2755c9de560dSAlex Tomas kfree(sbi->s_group_info); 2756c9de560dSAlex Tomas } 2757c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2758c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2759c9de560dSAlex Tomas if (sbi->s_buddy_cache) 2760c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2761c9de560dSAlex Tomas if (sbi->s_mb_stats) { 2762c9de560dSAlex Tomas printk(KERN_INFO 2763c9de560dSAlex Tomas "EXT4-fs: mballoc: %u blocks %u reqs (%u success)\n", 2764c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2765c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2766c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 2767c9de560dSAlex Tomas printk(KERN_INFO 2768c9de560dSAlex Tomas "EXT4-fs: mballoc: %u extents scanned, %u goal hits, " 2769c9de560dSAlex Tomas "%u 2^N hits, %u breaks, %u lost\n", 2770c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 2771c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 2772c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 2773c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 2774c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 2775c9de560dSAlex Tomas printk(KERN_INFO 2776c9de560dSAlex Tomas "EXT4-fs: mballoc: %lu generated and it took %Lu\n", 2777c9de560dSAlex Tomas sbi->s_mb_buddies_generated++, 2778c9de560dSAlex Tomas sbi->s_mb_generation_time); 2779c9de560dSAlex Tomas printk(KERN_INFO 2780c9de560dSAlex Tomas "EXT4-fs: mballoc: %u preallocated, %u discarded\n", 2781c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 2782c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 2783c9de560dSAlex Tomas } 2784c9de560dSAlex Tomas 2785c9de560dSAlex Tomas kfree(sbi->s_locality_groups); 2786c9de560dSAlex Tomas 2787c9de560dSAlex Tomas ext4_mb_history_release(sb); 2788c9de560dSAlex Tomas ext4_mb_destroy_per_dev_proc(sb); 2789c9de560dSAlex Tomas 2790c9de560dSAlex Tomas return 0; 2791c9de560dSAlex Tomas } 2792c9de560dSAlex Tomas 2793c9de560dSAlex Tomas static void ext4_mb_free_committed_blocks(struct super_block *sb) 2794c9de560dSAlex Tomas { 2795c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2796c9de560dSAlex Tomas int err; 2797c9de560dSAlex Tomas int i; 2798c9de560dSAlex Tomas int count = 0; 2799c9de560dSAlex Tomas int count2 = 0; 2800c9de560dSAlex Tomas struct ext4_free_metadata *md; 2801c9de560dSAlex Tomas struct ext4_buddy e4b; 2802c9de560dSAlex Tomas 2803c9de560dSAlex Tomas if (list_empty(&sbi->s_committed_transaction)) 2804c9de560dSAlex Tomas return; 2805c9de560dSAlex Tomas 2806c9de560dSAlex Tomas /* there is committed blocks to be freed yet */ 2807c9de560dSAlex Tomas do { 2808c9de560dSAlex Tomas /* get next array of blocks */ 2809c9de560dSAlex Tomas md = NULL; 2810c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2811c9de560dSAlex Tomas if (!list_empty(&sbi->s_committed_transaction)) { 2812c9de560dSAlex Tomas md = list_entry(sbi->s_committed_transaction.next, 2813c9de560dSAlex Tomas struct ext4_free_metadata, list); 2814c9de560dSAlex Tomas list_del(&md->list); 2815c9de560dSAlex Tomas } 2816c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2817c9de560dSAlex Tomas 2818c9de560dSAlex Tomas if (md == NULL) 2819c9de560dSAlex Tomas break; 2820c9de560dSAlex Tomas 2821c9de560dSAlex Tomas mb_debug("gonna free %u blocks in group %lu (0x%p):", 2822c9de560dSAlex Tomas md->num, md->group, md); 2823c9de560dSAlex Tomas 2824c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, md->group, &e4b); 2825c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 2826c9de560dSAlex Tomas BUG_ON(err != 0); 2827c9de560dSAlex Tomas 2828c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 2829c9de560dSAlex Tomas count += md->num; 2830c9de560dSAlex Tomas count2++; 2831c9de560dSAlex Tomas ext4_lock_group(sb, md->group); 2832c9de560dSAlex Tomas for (i = 0; i < md->num; i++) { 2833c9de560dSAlex Tomas mb_debug(" %u", md->blocks[i]); 2834c9de560dSAlex Tomas err = mb_free_blocks(NULL, &e4b, md->blocks[i], 1); 2835c9de560dSAlex Tomas BUG_ON(err != 0); 2836c9de560dSAlex Tomas } 2837c9de560dSAlex Tomas mb_debug("\n"); 2838c9de560dSAlex Tomas ext4_unlock_group(sb, md->group); 2839c9de560dSAlex Tomas 2840c9de560dSAlex Tomas /* balance refcounts from ext4_mb_free_metadata() */ 2841c9de560dSAlex Tomas page_cache_release(e4b.bd_buddy_page); 2842c9de560dSAlex Tomas page_cache_release(e4b.bd_bitmap_page); 2843c9de560dSAlex Tomas 2844c9de560dSAlex Tomas kfree(md); 2845c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 2846c9de560dSAlex Tomas 2847c9de560dSAlex Tomas } while (md); 2848c9de560dSAlex Tomas 2849c9de560dSAlex Tomas mb_debug("freed %u blocks in %u structures\n", count, count2); 2850c9de560dSAlex Tomas } 2851c9de560dSAlex Tomas 2852c9de560dSAlex Tomas #define EXT4_ROOT "ext4" 2853c9de560dSAlex Tomas #define EXT4_MB_STATS_NAME "stats" 2854c9de560dSAlex Tomas #define EXT4_MB_MAX_TO_SCAN_NAME "max_to_scan" 2855c9de560dSAlex Tomas #define EXT4_MB_MIN_TO_SCAN_NAME "min_to_scan" 2856c9de560dSAlex Tomas #define EXT4_MB_ORDER2_REQ "order2_req" 2857c9de560dSAlex Tomas #define EXT4_MB_STREAM_REQ "stream_req" 2858c9de560dSAlex Tomas #define EXT4_MB_GROUP_PREALLOC "group_prealloc" 2859c9de560dSAlex Tomas 2860c9de560dSAlex Tomas 2861c9de560dSAlex Tomas 2862c9de560dSAlex Tomas #define MB_PROC_VALUE_READ(name) \ 2863c9de560dSAlex Tomas static int ext4_mb_read_##name(char *page, char **start, \ 2864c9de560dSAlex Tomas off_t off, int count, int *eof, void *data) \ 2865c9de560dSAlex Tomas { \ 2866c9de560dSAlex Tomas struct ext4_sb_info *sbi = data; \ 2867c9de560dSAlex Tomas int len; \ 2868c9de560dSAlex Tomas *eof = 1; \ 2869c9de560dSAlex Tomas if (off != 0) \ 2870c9de560dSAlex Tomas return 0; \ 2871c9de560dSAlex Tomas len = sprintf(page, "%ld\n", sbi->s_mb_##name); \ 2872c9de560dSAlex Tomas *start = page; \ 2873c9de560dSAlex Tomas return len; \ 2874c9de560dSAlex Tomas } 2875c9de560dSAlex Tomas 2876c9de560dSAlex Tomas #define MB_PROC_VALUE_WRITE(name) \ 2877c9de560dSAlex Tomas static int ext4_mb_write_##name(struct file *file, \ 2878c9de560dSAlex Tomas const char __user *buf, unsigned long cnt, void *data) \ 2879c9de560dSAlex Tomas { \ 2880c9de560dSAlex Tomas struct ext4_sb_info *sbi = data; \ 2881c9de560dSAlex Tomas char str[32]; \ 2882c9de560dSAlex Tomas long value; \ 2883c9de560dSAlex Tomas if (cnt >= sizeof(str)) \ 2884c9de560dSAlex Tomas return -EINVAL; \ 2885c9de560dSAlex Tomas if (copy_from_user(str, buf, cnt)) \ 2886c9de560dSAlex Tomas return -EFAULT; \ 2887c9de560dSAlex Tomas value = simple_strtol(str, NULL, 0); \ 2888c9de560dSAlex Tomas if (value <= 0) \ 2889c9de560dSAlex Tomas return -ERANGE; \ 2890c9de560dSAlex Tomas sbi->s_mb_##name = value; \ 2891c9de560dSAlex Tomas return cnt; \ 2892c9de560dSAlex Tomas } 2893c9de560dSAlex Tomas 2894c9de560dSAlex Tomas MB_PROC_VALUE_READ(stats); 2895c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(stats); 2896c9de560dSAlex Tomas MB_PROC_VALUE_READ(max_to_scan); 2897c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(max_to_scan); 2898c9de560dSAlex Tomas MB_PROC_VALUE_READ(min_to_scan); 2899c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(min_to_scan); 2900c9de560dSAlex Tomas MB_PROC_VALUE_READ(order2_reqs); 2901c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(order2_reqs); 2902c9de560dSAlex Tomas MB_PROC_VALUE_READ(stream_request); 2903c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(stream_request); 2904c9de560dSAlex Tomas MB_PROC_VALUE_READ(group_prealloc); 2905c9de560dSAlex Tomas MB_PROC_VALUE_WRITE(group_prealloc); 2906c9de560dSAlex Tomas 2907c9de560dSAlex Tomas #define MB_PROC_HANDLER(name, var) \ 2908c9de560dSAlex Tomas do { \ 2909c9de560dSAlex Tomas proc = create_proc_entry(name, mode, sbi->s_mb_proc); \ 2910c9de560dSAlex Tomas if (proc == NULL) { \ 2911c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: can't to create %s\n", name); \ 2912c9de560dSAlex Tomas goto err_out; \ 2913c9de560dSAlex Tomas } \ 2914c9de560dSAlex Tomas proc->data = sbi; \ 2915c9de560dSAlex Tomas proc->read_proc = ext4_mb_read_##var ; \ 2916c9de560dSAlex Tomas proc->write_proc = ext4_mb_write_##var; \ 2917c9de560dSAlex Tomas } while (0) 2918c9de560dSAlex Tomas 2919c9de560dSAlex Tomas static int ext4_mb_init_per_dev_proc(struct super_block *sb) 2920c9de560dSAlex Tomas { 2921c9de560dSAlex Tomas mode_t mode = S_IFREG | S_IRUGO | S_IWUSR; 2922c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2923c9de560dSAlex Tomas struct proc_dir_entry *proc; 2924c9de560dSAlex Tomas char devname[64]; 2925c9de560dSAlex Tomas 2926c9de560dSAlex Tomas snprintf(devname, sizeof(devname) - 1, "%s", 2927c9de560dSAlex Tomas bdevname(sb->s_bdev, devname)); 2928c9de560dSAlex Tomas sbi->s_mb_proc = proc_mkdir(devname, proc_root_ext4); 2929c9de560dSAlex Tomas 2930c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_STATS_NAME, stats); 2931c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_MAX_TO_SCAN_NAME, max_to_scan); 2932c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_MIN_TO_SCAN_NAME, min_to_scan); 2933c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_ORDER2_REQ, order2_reqs); 2934c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_STREAM_REQ, stream_request); 2935c9de560dSAlex Tomas MB_PROC_HANDLER(EXT4_MB_GROUP_PREALLOC, group_prealloc); 2936c9de560dSAlex Tomas 2937c9de560dSAlex Tomas return 0; 2938c9de560dSAlex Tomas 2939c9de560dSAlex Tomas err_out: 2940c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: Unable to create %s\n", devname); 2941c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_GROUP_PREALLOC, sbi->s_mb_proc); 2942c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_STREAM_REQ, sbi->s_mb_proc); 2943c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_ORDER2_REQ, sbi->s_mb_proc); 2944c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_MIN_TO_SCAN_NAME, sbi->s_mb_proc); 2945c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_MAX_TO_SCAN_NAME, sbi->s_mb_proc); 2946c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_STATS_NAME, sbi->s_mb_proc); 2947c9de560dSAlex Tomas remove_proc_entry(devname, proc_root_ext4); 2948c9de560dSAlex Tomas sbi->s_mb_proc = NULL; 2949c9de560dSAlex Tomas 2950c9de560dSAlex Tomas return -ENOMEM; 2951c9de560dSAlex Tomas } 2952c9de560dSAlex Tomas 2953c9de560dSAlex Tomas static int ext4_mb_destroy_per_dev_proc(struct super_block *sb) 2954c9de560dSAlex Tomas { 2955c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2956c9de560dSAlex Tomas char devname[64]; 2957c9de560dSAlex Tomas 2958c9de560dSAlex Tomas if (sbi->s_mb_proc == NULL) 2959c9de560dSAlex Tomas return -EINVAL; 2960c9de560dSAlex Tomas 2961c9de560dSAlex Tomas snprintf(devname, sizeof(devname) - 1, "%s", 2962c9de560dSAlex Tomas bdevname(sb->s_bdev, devname)); 2963c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_GROUP_PREALLOC, sbi->s_mb_proc); 2964c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_STREAM_REQ, sbi->s_mb_proc); 2965c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_ORDER2_REQ, sbi->s_mb_proc); 2966c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_MIN_TO_SCAN_NAME, sbi->s_mb_proc); 2967c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_MAX_TO_SCAN_NAME, sbi->s_mb_proc); 2968c9de560dSAlex Tomas remove_proc_entry(EXT4_MB_STATS_NAME, sbi->s_mb_proc); 2969c9de560dSAlex Tomas remove_proc_entry(devname, proc_root_ext4); 2970c9de560dSAlex Tomas 2971c9de560dSAlex Tomas return 0; 2972c9de560dSAlex Tomas } 2973c9de560dSAlex Tomas 2974c9de560dSAlex Tomas int __init init_ext4_mballoc(void) 2975c9de560dSAlex Tomas { 2976c9de560dSAlex Tomas ext4_pspace_cachep = 2977c9de560dSAlex Tomas kmem_cache_create("ext4_prealloc_space", 2978c9de560dSAlex Tomas sizeof(struct ext4_prealloc_space), 2979c9de560dSAlex Tomas 0, SLAB_RECLAIM_ACCOUNT, NULL); 2980c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 2981c9de560dSAlex Tomas return -ENOMEM; 2982c9de560dSAlex Tomas 2983256bdb49SEric Sandeen ext4_ac_cachep = 2984256bdb49SEric Sandeen kmem_cache_create("ext4_alloc_context", 2985256bdb49SEric Sandeen sizeof(struct ext4_allocation_context), 2986256bdb49SEric Sandeen 0, SLAB_RECLAIM_ACCOUNT, NULL); 2987256bdb49SEric Sandeen if (ext4_ac_cachep == NULL) { 2988256bdb49SEric Sandeen kmem_cache_destroy(ext4_pspace_cachep); 2989256bdb49SEric Sandeen return -ENOMEM; 2990256bdb49SEric Sandeen } 2991c9de560dSAlex Tomas #ifdef CONFIG_PROC_FS 2992c9de560dSAlex Tomas proc_root_ext4 = proc_mkdir(EXT4_ROOT, proc_root_fs); 2993c9de560dSAlex Tomas if (proc_root_ext4 == NULL) 2994c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: Unable to create %s\n", EXT4_ROOT); 2995c9de560dSAlex Tomas #endif 2996c9de560dSAlex Tomas return 0; 2997c9de560dSAlex Tomas } 2998c9de560dSAlex Tomas 2999c9de560dSAlex Tomas void exit_ext4_mballoc(void) 3000c9de560dSAlex Tomas { 3001c9de560dSAlex Tomas /* XXX: synchronize_rcu(); */ 3002c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 3003256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 3004c9de560dSAlex Tomas #ifdef CONFIG_PROC_FS 3005c9de560dSAlex Tomas remove_proc_entry(EXT4_ROOT, proc_root_fs); 3006c9de560dSAlex Tomas #endif 3007c9de560dSAlex Tomas } 3008c9de560dSAlex Tomas 3009c9de560dSAlex Tomas 3010c9de560dSAlex Tomas /* 3011c9de560dSAlex Tomas * Check quota and mark choosed space (ac->ac_b_ex) non-free in bitmaps 3012c9de560dSAlex Tomas * Returns 0 if success or error code 3013c9de560dSAlex Tomas */ 3014c9de560dSAlex Tomas static int ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 3015c9de560dSAlex Tomas handle_t *handle) 3016c9de560dSAlex Tomas { 3017c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3018c9de560dSAlex Tomas struct ext4_super_block *es; 3019c9de560dSAlex Tomas struct ext4_group_desc *gdp; 3020c9de560dSAlex Tomas struct buffer_head *gdp_bh; 3021c9de560dSAlex Tomas struct ext4_sb_info *sbi; 3022c9de560dSAlex Tomas struct super_block *sb; 3023c9de560dSAlex Tomas ext4_fsblk_t block; 3024c9de560dSAlex Tomas int err; 3025c9de560dSAlex Tomas 3026c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3027c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 3028c9de560dSAlex Tomas 3029c9de560dSAlex Tomas sb = ac->ac_sb; 3030c9de560dSAlex Tomas sbi = EXT4_SB(sb); 3031c9de560dSAlex Tomas es = sbi->s_es; 3032c9de560dSAlex Tomas 3033c9de560dSAlex Tomas ext4_debug("using block group %lu(%d)\n", ac->ac_b_ex.fe_group, 3034c9de560dSAlex Tomas gdp->bg_free_blocks_count); 3035c9de560dSAlex Tomas 3036c9de560dSAlex Tomas err = -EIO; 3037c9de560dSAlex Tomas bitmap_bh = read_block_bitmap(sb, ac->ac_b_ex.fe_group); 3038c9de560dSAlex Tomas if (!bitmap_bh) 3039c9de560dSAlex Tomas goto out_err; 3040c9de560dSAlex Tomas 3041c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 3042c9de560dSAlex Tomas if (err) 3043c9de560dSAlex Tomas goto out_err; 3044c9de560dSAlex Tomas 3045c9de560dSAlex Tomas err = -EIO; 3046c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 3047c9de560dSAlex Tomas if (!gdp) 3048c9de560dSAlex Tomas goto out_err; 3049c9de560dSAlex Tomas 3050c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 3051c9de560dSAlex Tomas if (err) 3052c9de560dSAlex Tomas goto out_err; 3053c9de560dSAlex Tomas 3054c9de560dSAlex Tomas block = ac->ac_b_ex.fe_group * EXT4_BLOCKS_PER_GROUP(sb) 3055c9de560dSAlex Tomas + ac->ac_b_ex.fe_start 3056c9de560dSAlex Tomas + le32_to_cpu(es->s_first_data_block); 3057c9de560dSAlex Tomas 3058c9de560dSAlex Tomas if (block == ext4_block_bitmap(sb, gdp) || 3059c9de560dSAlex Tomas block == ext4_inode_bitmap(sb, gdp) || 3060c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 3061c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group)) { 3062c9de560dSAlex Tomas 3063c9de560dSAlex Tomas ext4_error(sb, __FUNCTION__, 3064c9de560dSAlex Tomas "Allocating block in system zone - block = %llu", 3065c9de560dSAlex Tomas block); 3066c9de560dSAlex Tomas } 3067c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 3068c9de560dSAlex Tomas { 3069c9de560dSAlex Tomas int i; 3070c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 3071c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 3072c9de560dSAlex Tomas bitmap_bh->b_data)); 3073c9de560dSAlex Tomas } 3074c9de560dSAlex Tomas } 3075c9de560dSAlex Tomas #endif 3076c9de560dSAlex Tomas mb_set_bits(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group), bitmap_bh->b_data, 3077c9de560dSAlex Tomas ac->ac_b_ex.fe_start, ac->ac_b_ex.fe_len); 3078c9de560dSAlex Tomas 3079c9de560dSAlex Tomas spin_lock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group)); 3080c9de560dSAlex Tomas if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 3081c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 3082c9de560dSAlex Tomas gdp->bg_free_blocks_count = 3083c9de560dSAlex Tomas cpu_to_le16(ext4_free_blocks_after_init(sb, 3084c9de560dSAlex Tomas ac->ac_b_ex.fe_group, 3085c9de560dSAlex Tomas gdp)); 3086c9de560dSAlex Tomas } 3087c9de560dSAlex Tomas gdp->bg_free_blocks_count = 3088c9de560dSAlex Tomas cpu_to_le16(le16_to_cpu(gdp->bg_free_blocks_count) 3089c9de560dSAlex Tomas - ac->ac_b_ex.fe_len); 3090c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); 3091c9de560dSAlex Tomas spin_unlock(sb_bgl_lock(sbi, ac->ac_b_ex.fe_group)); 3092c9de560dSAlex Tomas percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len); 3093c9de560dSAlex Tomas 3094c9de560dSAlex Tomas err = ext4_journal_dirty_metadata(handle, bitmap_bh); 3095c9de560dSAlex Tomas if (err) 3096c9de560dSAlex Tomas goto out_err; 3097c9de560dSAlex Tomas err = ext4_journal_dirty_metadata(handle, gdp_bh); 3098c9de560dSAlex Tomas 3099c9de560dSAlex Tomas out_err: 3100c9de560dSAlex Tomas sb->s_dirt = 1; 310142a10addSAneesh Kumar K.V brelse(bitmap_bh); 3102c9de560dSAlex Tomas return err; 3103c9de560dSAlex Tomas } 3104c9de560dSAlex Tomas 3105c9de560dSAlex Tomas /* 3106c9de560dSAlex Tomas * here we normalize request for locality group 3107c9de560dSAlex Tomas * Group request are normalized to s_strip size if we set the same via mount 3108c9de560dSAlex Tomas * option. If not we set it to s_mb_group_prealloc which can be configured via 3109c9de560dSAlex Tomas * /proc/fs/ext4/<partition>/group_prealloc 3110c9de560dSAlex Tomas * 3111c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 3112c9de560dSAlex Tomas */ 3113c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 3114c9de560dSAlex Tomas { 3115c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3116c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 3117c9de560dSAlex Tomas 3118c9de560dSAlex Tomas BUG_ON(lg == NULL); 3119c9de560dSAlex Tomas if (EXT4_SB(sb)->s_stripe) 3120c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_stripe; 3121c9de560dSAlex Tomas else 3122c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 3123c9de560dSAlex Tomas mb_debug("#%u: goal %lu blocks for locality group\n", 3124c9de560dSAlex Tomas current->pid, ac->ac_g_ex.fe_len); 3125c9de560dSAlex Tomas } 3126c9de560dSAlex Tomas 3127c9de560dSAlex Tomas /* 3128c9de560dSAlex Tomas * Normalization means making request better in terms of 3129c9de560dSAlex Tomas * size and alignment 3130c9de560dSAlex Tomas */ 3131c9de560dSAlex Tomas static void ext4_mb_normalize_request(struct ext4_allocation_context *ac, 3132c9de560dSAlex Tomas struct ext4_allocation_request *ar) 3133c9de560dSAlex Tomas { 3134c9de560dSAlex Tomas int bsbits, max; 3135c9de560dSAlex Tomas ext4_lblk_t end; 3136c9de560dSAlex Tomas struct list_head *cur; 3137c9de560dSAlex Tomas loff_t size, orig_size, start_off; 3138c9de560dSAlex Tomas ext4_lblk_t start, orig_start; 3139c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3140c9de560dSAlex Tomas 3141c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 3142c9de560dSAlex Tomas do not need preallocation */ 3143c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3144c9de560dSAlex Tomas return; 3145c9de560dSAlex Tomas 3146c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 3147c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 3148c9de560dSAlex Tomas return; 3149c9de560dSAlex Tomas 3150c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 3151c9de560dSAlex Tomas * required (it's a tail, for example) */ 3152c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 3153c9de560dSAlex Tomas return; 3154c9de560dSAlex Tomas 3155c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 3156c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 3157c9de560dSAlex Tomas return ; 3158c9de560dSAlex Tomas } 3159c9de560dSAlex Tomas 3160c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 3161c9de560dSAlex Tomas 3162c9de560dSAlex Tomas /* first, let's learn actual file size 3163c9de560dSAlex Tomas * given current request is allocated */ 3164c9de560dSAlex Tomas size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len; 3165c9de560dSAlex Tomas size = size << bsbits; 3166c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 3167c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 3168c9de560dSAlex Tomas 3169c9de560dSAlex Tomas /* max available blocks in a free group */ 3170c9de560dSAlex Tomas max = EXT4_BLOCKS_PER_GROUP(ac->ac_sb) - 1 - 1 - 3171c9de560dSAlex Tomas EXT4_SB(ac->ac_sb)->s_itb_per_group; 3172c9de560dSAlex Tomas 3173c9de560dSAlex Tomas #define NRL_CHECK_SIZE(req, size, max,bits) \ 3174c9de560dSAlex Tomas (req <= (size) || max <= ((size) >> bits)) 3175c9de560dSAlex Tomas 3176c9de560dSAlex Tomas /* first, try to predict filesize */ 3177c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 3178c9de560dSAlex Tomas start_off = 0; 3179c9de560dSAlex Tomas if (size <= 16 * 1024) { 3180c9de560dSAlex Tomas size = 16 * 1024; 3181c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 3182c9de560dSAlex Tomas size = 32 * 1024; 3183c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 3184c9de560dSAlex Tomas size = 64 * 1024; 3185c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 3186c9de560dSAlex Tomas size = 128 * 1024; 3187c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 3188c9de560dSAlex Tomas size = 256 * 1024; 3189c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 3190c9de560dSAlex Tomas size = 512 * 1024; 3191c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 3192c9de560dSAlex Tomas size = 1024 * 1024; 3193c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, bsbits)) { 3194c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3195c9de560dSAlex Tomas (20 - bsbits)) << 20; 3196c9de560dSAlex Tomas size = 1024 * 1024; 3197c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, bsbits)) { 3198c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3199c9de560dSAlex Tomas (22 - bsbits)) << 22; 3200c9de560dSAlex Tomas size = 4 * 1024 * 1024; 3201c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 3202c9de560dSAlex Tomas (8<<20)>>bsbits, max, bsbits)) { 3203c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 3204c9de560dSAlex Tomas (23 - bsbits)) << 23; 3205c9de560dSAlex Tomas size = 8 * 1024 * 1024; 3206c9de560dSAlex Tomas } else { 3207c9de560dSAlex Tomas start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits; 3208c9de560dSAlex Tomas size = ac->ac_o_ex.fe_len << bsbits; 3209c9de560dSAlex Tomas } 3210c9de560dSAlex Tomas orig_size = size = size >> bsbits; 3211c9de560dSAlex Tomas orig_start = start = start_off >> bsbits; 3212c9de560dSAlex Tomas 3213c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 3214c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 3215c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 3216c9de560dSAlex Tomas start = ar->lleft + 1; 3217c9de560dSAlex Tomas } 3218c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 3219c9de560dSAlex Tomas size -= start + size - ar->lright; 3220c9de560dSAlex Tomas 3221c9de560dSAlex Tomas end = start + size; 3222c9de560dSAlex Tomas 3223c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 3224c9de560dSAlex Tomas rcu_read_lock(); 3225c9de560dSAlex Tomas list_for_each_rcu(cur, &ei->i_prealloc_list) { 3226c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3227c9de560dSAlex Tomas unsigned long pa_end; 3228c9de560dSAlex Tomas 3229c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list); 3230c9de560dSAlex Tomas 3231c9de560dSAlex Tomas if (pa->pa_deleted) 3232c9de560dSAlex Tomas continue; 3233c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3234c9de560dSAlex Tomas if (pa->pa_deleted) { 3235c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3236c9de560dSAlex Tomas continue; 3237c9de560dSAlex Tomas } 3238c9de560dSAlex Tomas 3239c9de560dSAlex Tomas pa_end = pa->pa_lstart + pa->pa_len; 3240c9de560dSAlex Tomas 3241c9de560dSAlex Tomas /* PA must not overlap original request */ 3242c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 3243c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 3244c9de560dSAlex Tomas 3245c9de560dSAlex Tomas /* skip PA normalized request doesn't overlap with */ 3246c9de560dSAlex Tomas if (pa->pa_lstart >= end) { 3247c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3248c9de560dSAlex Tomas continue; 3249c9de560dSAlex Tomas } 3250c9de560dSAlex Tomas if (pa_end <= start) { 3251c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3252c9de560dSAlex Tomas continue; 3253c9de560dSAlex Tomas } 3254c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3255c9de560dSAlex Tomas 3256c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3257c9de560dSAlex Tomas BUG_ON(pa_end < start); 3258c9de560dSAlex Tomas start = pa_end; 3259c9de560dSAlex Tomas } 3260c9de560dSAlex Tomas 3261c9de560dSAlex Tomas if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3262c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3263c9de560dSAlex Tomas end = pa->pa_lstart; 3264c9de560dSAlex Tomas } 3265c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3266c9de560dSAlex Tomas } 3267c9de560dSAlex Tomas rcu_read_unlock(); 3268c9de560dSAlex Tomas size = end - start; 3269c9de560dSAlex Tomas 3270c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3271c9de560dSAlex Tomas rcu_read_lock(); 3272c9de560dSAlex Tomas list_for_each_rcu(cur, &ei->i_prealloc_list) { 3273c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3274c9de560dSAlex Tomas unsigned long pa_end; 3275c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list); 3276c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3277c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3278c9de560dSAlex Tomas pa_end = pa->pa_lstart + pa->pa_len; 3279c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3280c9de560dSAlex Tomas } 3281c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3282c9de560dSAlex Tomas } 3283c9de560dSAlex Tomas rcu_read_unlock(); 3284c9de560dSAlex Tomas 3285c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3286c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 3287c9de560dSAlex Tomas printk(KERN_ERR "start %lu, size %lu, fe_logical %lu\n", 3288c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3289c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3290c9de560dSAlex Tomas } 3291c9de560dSAlex Tomas BUG_ON(start + size <= ac->ac_o_ex.fe_logical && 3292c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical); 3293c9de560dSAlex Tomas BUG_ON(size <= 0 || size >= EXT4_BLOCKS_PER_GROUP(ac->ac_sb)); 3294c9de560dSAlex Tomas 3295c9de560dSAlex Tomas /* now prepare goal request */ 3296c9de560dSAlex Tomas 3297c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3298c9de560dSAlex Tomas * placement or satisfy big request as is */ 3299c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 3300c9de560dSAlex Tomas ac->ac_g_ex.fe_len = size; 3301c9de560dSAlex Tomas 3302c9de560dSAlex Tomas /* define goal start in order to merge */ 3303c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3304c9de560dSAlex Tomas /* merge to the right */ 3305c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3306c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3307c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3308c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3309c9de560dSAlex Tomas } 3310c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3311c9de560dSAlex Tomas /* merge to the left */ 3312c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3313c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3314c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3315c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3316c9de560dSAlex Tomas } 3317c9de560dSAlex Tomas 3318c9de560dSAlex Tomas mb_debug("goal: %u(was %u) blocks at %u\n", (unsigned) size, 3319c9de560dSAlex Tomas (unsigned) orig_size, (unsigned) start); 3320c9de560dSAlex Tomas } 3321c9de560dSAlex Tomas 3322c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3323c9de560dSAlex Tomas { 3324c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3325c9de560dSAlex Tomas 3326c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3327c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3328c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3329c9de560dSAlex Tomas if (ac->ac_o_ex.fe_len >= ac->ac_g_ex.fe_len) 3330c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3331c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3332c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3333c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3334c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3335c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3336c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3337c9de560dSAlex Tomas } 3338c9de560dSAlex Tomas 3339c9de560dSAlex Tomas ext4_mb_store_history(ac); 3340c9de560dSAlex Tomas } 3341c9de560dSAlex Tomas 3342c9de560dSAlex Tomas /* 3343c9de560dSAlex Tomas * use blocks preallocated to inode 3344c9de560dSAlex Tomas */ 3345c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3346c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3347c9de560dSAlex Tomas { 3348c9de560dSAlex Tomas ext4_fsblk_t start; 3349c9de560dSAlex Tomas ext4_fsblk_t end; 3350c9de560dSAlex Tomas int len; 3351c9de560dSAlex Tomas 3352c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3353c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 3354c9de560dSAlex Tomas end = min(pa->pa_pstart + pa->pa_len, start + ac->ac_o_ex.fe_len); 3355c9de560dSAlex Tomas len = end - start; 3356c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3357c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3358c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3359c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3360c9de560dSAlex Tomas ac->ac_pa = pa; 3361c9de560dSAlex Tomas 3362c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 3363c9de560dSAlex Tomas BUG_ON(start + len > pa->pa_pstart + pa->pa_len); 3364c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3365c9de560dSAlex Tomas pa->pa_free -= len; 3366c9de560dSAlex Tomas 3367c9de560dSAlex Tomas mb_debug("use %llu/%lu from inode pa %p\n", start, len, pa); 3368c9de560dSAlex Tomas } 3369c9de560dSAlex Tomas 3370c9de560dSAlex Tomas /* 3371c9de560dSAlex Tomas * use blocks preallocated to locality group 3372c9de560dSAlex Tomas */ 3373c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3374c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3375c9de560dSAlex Tomas { 3376c9de560dSAlex Tomas unsigned len = ac->ac_o_ex.fe_len; 3377c9de560dSAlex Tomas 3378c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3379c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3380c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3381c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3382c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3383c9de560dSAlex Tomas ac->ac_pa = pa; 3384c9de560dSAlex Tomas 3385c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 338626346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3387c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 338826346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 338926346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3390c9de560dSAlex Tomas */ 3391c9de560dSAlex Tomas mb_debug("use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa); 3392c9de560dSAlex Tomas } 3393c9de560dSAlex Tomas 3394c9de560dSAlex Tomas /* 3395c9de560dSAlex Tomas * search goal blocks in preallocated space 3396c9de560dSAlex Tomas */ 3397c9de560dSAlex Tomas static int ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3398c9de560dSAlex Tomas { 3399c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3400c9de560dSAlex Tomas struct ext4_locality_group *lg; 3401c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3402c9de560dSAlex Tomas struct list_head *cur; 3403c9de560dSAlex Tomas 3404c9de560dSAlex Tomas /* only data can be preallocated */ 3405c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3406c9de560dSAlex Tomas return 0; 3407c9de560dSAlex Tomas 3408c9de560dSAlex Tomas /* first, try per-file preallocation */ 3409c9de560dSAlex Tomas rcu_read_lock(); 3410c9de560dSAlex Tomas list_for_each_rcu(cur, &ei->i_prealloc_list) { 3411c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list); 3412c9de560dSAlex Tomas 3413c9de560dSAlex Tomas /* all fields in this condition don't change, 3414c9de560dSAlex Tomas * so we can skip locking for them */ 3415c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 3416c9de560dSAlex Tomas ac->ac_o_ex.fe_logical >= pa->pa_lstart + pa->pa_len) 3417c9de560dSAlex Tomas continue; 3418c9de560dSAlex Tomas 3419c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3420c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3421c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3422c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3423c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3424c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3425c9de560dSAlex Tomas ac->ac_criteria = 10; 3426c9de560dSAlex Tomas rcu_read_unlock(); 3427c9de560dSAlex Tomas return 1; 3428c9de560dSAlex Tomas } 3429c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3430c9de560dSAlex Tomas } 3431c9de560dSAlex Tomas rcu_read_unlock(); 3432c9de560dSAlex Tomas 3433c9de560dSAlex Tomas /* can we use group allocation? */ 3434c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 3435c9de560dSAlex Tomas return 0; 3436c9de560dSAlex Tomas 3437c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3438c9de560dSAlex Tomas lg = ac->ac_lg; 3439c9de560dSAlex Tomas if (lg == NULL) 3440c9de560dSAlex Tomas return 0; 3441c9de560dSAlex Tomas 3442c9de560dSAlex Tomas rcu_read_lock(); 3443c9de560dSAlex Tomas list_for_each_rcu(cur, &lg->lg_prealloc_list) { 3444c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_inode_list); 3445c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3446c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free >= ac->ac_o_ex.fe_len) { 3447c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3448c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3449c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3450c9de560dSAlex Tomas ac->ac_criteria = 20; 3451c9de560dSAlex Tomas rcu_read_unlock(); 3452c9de560dSAlex Tomas return 1; 3453c9de560dSAlex Tomas } 3454c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3455c9de560dSAlex Tomas } 3456c9de560dSAlex Tomas rcu_read_unlock(); 3457c9de560dSAlex Tomas 3458c9de560dSAlex Tomas return 0; 3459c9de560dSAlex Tomas } 3460c9de560dSAlex Tomas 3461c9de560dSAlex Tomas /* 3462c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3463c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3464c9de560dSAlex Tomas * Need to be called with ext4 group lock (ext4_lock_group) 3465c9de560dSAlex Tomas */ 3466c9de560dSAlex Tomas static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3467c9de560dSAlex Tomas ext4_group_t group) 3468c9de560dSAlex Tomas { 3469c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3470c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3471c9de560dSAlex Tomas struct list_head *cur; 3472c9de560dSAlex Tomas ext4_group_t groupnr; 3473c9de560dSAlex Tomas ext4_grpblk_t start; 3474c9de560dSAlex Tomas int preallocated = 0; 3475c9de560dSAlex Tomas int count = 0; 3476c9de560dSAlex Tomas int len; 3477c9de560dSAlex Tomas 3478c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3479c9de560dSAlex Tomas * so the only competing code is preallocation use. 3480c9de560dSAlex Tomas * we don't need any locking here 3481c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3482c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3483c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3484c9de560dSAlex Tomas * is dropping preallocation 3485c9de560dSAlex Tomas */ 3486c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3487c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3488c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3489c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3490c9de560dSAlex Tomas &groupnr, &start); 3491c9de560dSAlex Tomas len = pa->pa_len; 3492c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3493c9de560dSAlex Tomas if (unlikely(len == 0)) 3494c9de560dSAlex Tomas continue; 3495c9de560dSAlex Tomas BUG_ON(groupnr != group); 3496c9de560dSAlex Tomas mb_set_bits(sb_bgl_lock(EXT4_SB(sb), group), 3497c9de560dSAlex Tomas bitmap, start, len); 3498c9de560dSAlex Tomas preallocated += len; 3499c9de560dSAlex Tomas count++; 3500c9de560dSAlex Tomas } 3501c9de560dSAlex Tomas mb_debug("prellocated %u for group %lu\n", preallocated, group); 3502c9de560dSAlex Tomas } 3503c9de560dSAlex Tomas 3504c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3505c9de560dSAlex Tomas { 3506c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3507c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 3508c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3509c9de560dSAlex Tomas } 3510c9de560dSAlex Tomas 3511c9de560dSAlex Tomas /* 3512c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3513c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3514c9de560dSAlex Tomas */ 3515c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3516c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3517c9de560dSAlex Tomas { 3518c9de560dSAlex Tomas unsigned long grp; 3519c9de560dSAlex Tomas 3520c9de560dSAlex Tomas if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) 3521c9de560dSAlex Tomas return; 3522c9de560dSAlex Tomas 3523c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3524c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3525c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3526c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3527c9de560dSAlex Tomas return; 3528c9de560dSAlex Tomas } 3529c9de560dSAlex Tomas 3530c9de560dSAlex Tomas pa->pa_deleted = 1; 3531c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3532c9de560dSAlex Tomas 3533c9de560dSAlex Tomas /* -1 is to protect from crossing allocation group */ 3534c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart - 1, &grp, NULL); 3535c9de560dSAlex Tomas 3536c9de560dSAlex Tomas /* 3537c9de560dSAlex Tomas * possible race: 3538c9de560dSAlex Tomas * 3539c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3540c9de560dSAlex Tomas * find block B in PA 3541c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3542c9de560dSAlex Tomas * mark B in on-disk bitmap 3543c9de560dSAlex Tomas * drop PA from group 3544c9de560dSAlex Tomas * mark all PAs in buddy 3545c9de560dSAlex Tomas * 3546c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3547c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3548c9de560dSAlex Tomas * against that pair 3549c9de560dSAlex Tomas */ 3550c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3551c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3552c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3553c9de560dSAlex Tomas 3554c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3555c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3556c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3557c9de560dSAlex Tomas 3558c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3559c9de560dSAlex Tomas } 3560c9de560dSAlex Tomas 3561c9de560dSAlex Tomas /* 3562c9de560dSAlex Tomas * creates new preallocated space for given inode 3563c9de560dSAlex Tomas */ 3564c9de560dSAlex Tomas static int ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3565c9de560dSAlex Tomas { 3566c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3567c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3568c9de560dSAlex Tomas struct ext4_group_info *grp; 3569c9de560dSAlex Tomas struct ext4_inode_info *ei; 3570c9de560dSAlex Tomas 3571c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3572c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3573c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3574c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3575c9de560dSAlex Tomas 3576c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3577c9de560dSAlex Tomas if (pa == NULL) 3578c9de560dSAlex Tomas return -ENOMEM; 3579c9de560dSAlex Tomas 3580c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3581c9de560dSAlex Tomas int winl; 3582c9de560dSAlex Tomas int wins; 3583c9de560dSAlex Tomas int win; 3584c9de560dSAlex Tomas int offs; 3585c9de560dSAlex Tomas 3586c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3587c9de560dSAlex Tomas * so, found space must get proper lstart 3588c9de560dSAlex Tomas * to cover original request */ 3589c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3590c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3591c9de560dSAlex Tomas 3592c9de560dSAlex Tomas /* we're limited by original request in that 3593c9de560dSAlex Tomas * logical block must be covered any way 3594c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3595c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3596c9de560dSAlex Tomas 3597c9de560dSAlex Tomas /* also, we should cover whole original request */ 3598c9de560dSAlex Tomas wins = ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len; 3599c9de560dSAlex Tomas 3600c9de560dSAlex Tomas /* the smallest one defines real window */ 3601c9de560dSAlex Tomas win = min(winl, wins); 3602c9de560dSAlex Tomas 3603c9de560dSAlex Tomas offs = ac->ac_o_ex.fe_logical % ac->ac_b_ex.fe_len; 3604c9de560dSAlex Tomas if (offs && offs < win) 3605c9de560dSAlex Tomas win = offs; 3606c9de560dSAlex Tomas 3607c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - win; 3608c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3609c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3610c9de560dSAlex Tomas } 3611c9de560dSAlex Tomas 3612c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3613c9de560dSAlex Tomas * allocated blocks for history */ 3614c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3615c9de560dSAlex Tomas 3616c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3617c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3618c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3619c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3620c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3621c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3622c9de560dSAlex Tomas pa->pa_deleted = 0; 3623c9de560dSAlex Tomas pa->pa_linear = 0; 3624c9de560dSAlex Tomas 3625c9de560dSAlex Tomas mb_debug("new inode pa %p: %llu/%u for %u\n", pa, 3626c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 3627c9de560dSAlex Tomas 3628c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3629c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3630c9de560dSAlex Tomas 3631c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3632c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3633c9de560dSAlex Tomas 3634c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3635c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3636c9de560dSAlex Tomas 3637c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3638c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3639c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3640c9de560dSAlex Tomas 3641c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3642c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3643c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3644c9de560dSAlex Tomas 3645c9de560dSAlex Tomas return 0; 3646c9de560dSAlex Tomas } 3647c9de560dSAlex Tomas 3648c9de560dSAlex Tomas /* 3649c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3650c9de560dSAlex Tomas */ 3651c9de560dSAlex Tomas static int ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3652c9de560dSAlex Tomas { 3653c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3654c9de560dSAlex Tomas struct ext4_locality_group *lg; 3655c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3656c9de560dSAlex Tomas struct ext4_group_info *grp; 3657c9de560dSAlex Tomas 3658c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3659c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3660c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3661c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3662c9de560dSAlex Tomas 3663c9de560dSAlex Tomas BUG_ON(ext4_pspace_cachep == NULL); 3664c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3665c9de560dSAlex Tomas if (pa == NULL) 3666c9de560dSAlex Tomas return -ENOMEM; 3667c9de560dSAlex Tomas 3668c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3669c9de560dSAlex Tomas * allocated blocks for history */ 3670c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3671c9de560dSAlex Tomas 3672c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3673c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 3674c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3675c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3676c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3677c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3678c9de560dSAlex Tomas pa->pa_deleted = 0; 3679c9de560dSAlex Tomas pa->pa_linear = 1; 3680c9de560dSAlex Tomas 3681c9de560dSAlex Tomas mb_debug("new group pa %p: %llu/%u for %u\n", pa, 3682c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 3683c9de560dSAlex Tomas 3684c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3685c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3686c9de560dSAlex Tomas 3687c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3688c9de560dSAlex Tomas lg = ac->ac_lg; 3689c9de560dSAlex Tomas BUG_ON(lg == NULL); 3690c9de560dSAlex Tomas 3691c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 3692c9de560dSAlex Tomas pa->pa_inode = NULL; 3693c9de560dSAlex Tomas 3694c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3695c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3696c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3697c9de560dSAlex Tomas 3698c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3699c9de560dSAlex Tomas list_add_tail_rcu(&pa->pa_inode_list, &lg->lg_prealloc_list); 3700c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3701c9de560dSAlex Tomas 3702c9de560dSAlex Tomas return 0; 3703c9de560dSAlex Tomas } 3704c9de560dSAlex Tomas 3705c9de560dSAlex Tomas static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 3706c9de560dSAlex Tomas { 3707c9de560dSAlex Tomas int err; 3708c9de560dSAlex Tomas 3709c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 3710c9de560dSAlex Tomas err = ext4_mb_new_group_pa(ac); 3711c9de560dSAlex Tomas else 3712c9de560dSAlex Tomas err = ext4_mb_new_inode_pa(ac); 3713c9de560dSAlex Tomas return err; 3714c9de560dSAlex Tomas } 3715c9de560dSAlex Tomas 3716c9de560dSAlex Tomas /* 3717c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 3718c9de560dSAlex Tomas * in-core bitmap and buddy. 3719c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 3720c9de560dSAlex Tomas * nobody else can find/use it. 3721c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 3722c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 3723c9de560dSAlex Tomas */ 3724c9de560dSAlex Tomas static int ext4_mb_release_inode_pa(struct ext4_buddy *e4b, 3725c9de560dSAlex Tomas struct buffer_head *bitmap_bh, 3726c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3727c9de560dSAlex Tomas { 3728256bdb49SEric Sandeen struct ext4_allocation_context *ac; 3729c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3730c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3731c9de560dSAlex Tomas unsigned long end; 3732c9de560dSAlex Tomas unsigned long next; 3733c9de560dSAlex Tomas ext4_group_t group; 3734c9de560dSAlex Tomas ext4_grpblk_t bit; 3735c9de560dSAlex Tomas sector_t start; 3736c9de560dSAlex Tomas int err = 0; 3737c9de560dSAlex Tomas int free = 0; 3738c9de560dSAlex Tomas 3739c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3740c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3741c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3742c9de560dSAlex Tomas end = bit + pa->pa_len; 3743c9de560dSAlex Tomas 3744256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 3745256bdb49SEric Sandeen 3746256bdb49SEric Sandeen if (ac) { 3747256bdb49SEric Sandeen ac->ac_sb = sb; 3748256bdb49SEric Sandeen ac->ac_inode = pa->pa_inode; 3749256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_DISCARD; 3750256bdb49SEric Sandeen } 3751c9de560dSAlex Tomas 3752c9de560dSAlex Tomas while (bit < end) { 3753c9de560dSAlex Tomas bit = ext4_find_next_zero_bit(bitmap_bh->b_data, end, bit); 3754c9de560dSAlex Tomas if (bit >= end) 3755c9de560dSAlex Tomas break; 3756c9de560dSAlex Tomas next = ext4_find_next_bit(bitmap_bh->b_data, end, bit); 3757c9de560dSAlex Tomas if (next > end) 3758c9de560dSAlex Tomas next = end; 3759c9de560dSAlex Tomas start = group * EXT4_BLOCKS_PER_GROUP(sb) + bit + 3760c9de560dSAlex Tomas le32_to_cpu(sbi->s_es->s_first_data_block); 3761c9de560dSAlex Tomas mb_debug(" free preallocated %u/%u in group %u\n", 3762c9de560dSAlex Tomas (unsigned) start, (unsigned) next - bit, 3763c9de560dSAlex Tomas (unsigned) group); 3764c9de560dSAlex Tomas free += next - bit; 3765c9de560dSAlex Tomas 3766256bdb49SEric Sandeen if (ac) { 3767256bdb49SEric Sandeen ac->ac_b_ex.fe_group = group; 3768256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 3769256bdb49SEric Sandeen ac->ac_b_ex.fe_len = next - bit; 3770256bdb49SEric Sandeen ac->ac_b_ex.fe_logical = 0; 3771256bdb49SEric Sandeen ext4_mb_store_history(ac); 3772256bdb49SEric Sandeen } 3773c9de560dSAlex Tomas 3774c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 3775c9de560dSAlex Tomas bit = next + 1; 3776c9de560dSAlex Tomas } 3777c9de560dSAlex Tomas if (free != pa->pa_free) { 377826346ff6SAneesh Kumar K.V printk(KERN_CRIT "pa %p: logic %lu, phys. %lu, len %lu\n", 3779c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 3780c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 3781c9de560dSAlex Tomas (unsigned long) pa->pa_len); 378226346ff6SAneesh Kumar K.V ext4_error(sb, __FUNCTION__, "free %u, pa_free %u\n", 378326346ff6SAneesh Kumar K.V free, pa->pa_free); 3784e56eb659SAneesh Kumar K.V /* 3785e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 3786e56eb659SAneesh Kumar K.V * from the bitmap and continue. 3787e56eb659SAneesh Kumar K.V */ 3788c9de560dSAlex Tomas } 3789c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 3790256bdb49SEric Sandeen if (ac) 3791256bdb49SEric Sandeen kmem_cache_free(ext4_ac_cachep, ac); 3792c9de560dSAlex Tomas 3793c9de560dSAlex Tomas return err; 3794c9de560dSAlex Tomas } 3795c9de560dSAlex Tomas 3796c9de560dSAlex Tomas static int ext4_mb_release_group_pa(struct ext4_buddy *e4b, 3797c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3798c9de560dSAlex Tomas { 3799256bdb49SEric Sandeen struct ext4_allocation_context *ac; 3800c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3801c9de560dSAlex Tomas ext4_group_t group; 3802c9de560dSAlex Tomas ext4_grpblk_t bit; 3803c9de560dSAlex Tomas 3804256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 3805256bdb49SEric Sandeen 3806256bdb49SEric Sandeen if (ac) 3807256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_DISCARD; 3808c9de560dSAlex Tomas 3809c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3810c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3811c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3812c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 3813c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 3814c9de560dSAlex Tomas 3815256bdb49SEric Sandeen if (ac) { 3816256bdb49SEric Sandeen ac->ac_sb = sb; 3817256bdb49SEric Sandeen ac->ac_inode = NULL; 3818256bdb49SEric Sandeen ac->ac_b_ex.fe_group = group; 3819256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 3820256bdb49SEric Sandeen ac->ac_b_ex.fe_len = pa->pa_len; 3821256bdb49SEric Sandeen ac->ac_b_ex.fe_logical = 0; 3822256bdb49SEric Sandeen ext4_mb_store_history(ac); 3823256bdb49SEric Sandeen kmem_cache_free(ext4_ac_cachep, ac); 3824256bdb49SEric Sandeen } 3825c9de560dSAlex Tomas 3826c9de560dSAlex Tomas return 0; 3827c9de560dSAlex Tomas } 3828c9de560dSAlex Tomas 3829c9de560dSAlex Tomas /* 3830c9de560dSAlex Tomas * releases all preallocations in given group 3831c9de560dSAlex Tomas * 3832c9de560dSAlex Tomas * first, we need to decide discard policy: 3833c9de560dSAlex Tomas * - when do we discard 3834c9de560dSAlex Tomas * 1) ENOSPC 3835c9de560dSAlex Tomas * - how many do we discard 3836c9de560dSAlex Tomas * 1) how many requested 3837c9de560dSAlex Tomas */ 3838c9de560dSAlex Tomas static int ext4_mb_discard_group_preallocations(struct super_block *sb, 3839c9de560dSAlex Tomas ext4_group_t group, int needed) 3840c9de560dSAlex Tomas { 3841c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3842c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3843c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3844c9de560dSAlex Tomas struct list_head list; 3845c9de560dSAlex Tomas struct ext4_buddy e4b; 3846c9de560dSAlex Tomas int err; 3847c9de560dSAlex Tomas int busy = 0; 3848c9de560dSAlex Tomas int free = 0; 3849c9de560dSAlex Tomas 3850c9de560dSAlex Tomas mb_debug("discard preallocation for group %lu\n", group); 3851c9de560dSAlex Tomas 3852c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 3853c9de560dSAlex Tomas return 0; 3854c9de560dSAlex Tomas 3855c9de560dSAlex Tomas bitmap_bh = read_block_bitmap(sb, group); 3856c9de560dSAlex Tomas if (bitmap_bh == NULL) { 3857c9de560dSAlex Tomas /* error handling here */ 3858c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 3859c9de560dSAlex Tomas BUG_ON(bitmap_bh == NULL); 3860c9de560dSAlex Tomas } 3861c9de560dSAlex Tomas 3862c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3863c9de560dSAlex Tomas BUG_ON(err != 0); /* error handling here */ 3864c9de560dSAlex Tomas 3865c9de560dSAlex Tomas if (needed == 0) 3866c9de560dSAlex Tomas needed = EXT4_BLOCKS_PER_GROUP(sb) + 1; 3867c9de560dSAlex Tomas 3868c9de560dSAlex Tomas grp = ext4_get_group_info(sb, group); 3869c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3870c9de560dSAlex Tomas 3871c9de560dSAlex Tomas repeat: 3872c9de560dSAlex Tomas ext4_lock_group(sb, group); 3873c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 3874c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 3875c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3876c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3877c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3878c9de560dSAlex Tomas busy = 1; 3879c9de560dSAlex Tomas continue; 3880c9de560dSAlex Tomas } 3881c9de560dSAlex Tomas if (pa->pa_deleted) { 3882c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3883c9de560dSAlex Tomas continue; 3884c9de560dSAlex Tomas } 3885c9de560dSAlex Tomas 3886c9de560dSAlex Tomas /* seems this one can be freed ... */ 3887c9de560dSAlex Tomas pa->pa_deleted = 1; 3888c9de560dSAlex Tomas 3889c9de560dSAlex Tomas /* we can trust pa_free ... */ 3890c9de560dSAlex Tomas free += pa->pa_free; 3891c9de560dSAlex Tomas 3892c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3893c9de560dSAlex Tomas 3894c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3895c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3896c9de560dSAlex Tomas } 3897c9de560dSAlex Tomas 3898c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 3899c9de560dSAlex Tomas if (free < needed && busy) { 3900c9de560dSAlex Tomas busy = 0; 3901c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3902c9de560dSAlex Tomas /* 3903c9de560dSAlex Tomas * Yield the CPU here so that we don't get soft lockup 3904c9de560dSAlex Tomas * in non preempt case. 3905c9de560dSAlex Tomas */ 3906c9de560dSAlex Tomas yield(); 3907c9de560dSAlex Tomas goto repeat; 3908c9de560dSAlex Tomas } 3909c9de560dSAlex Tomas 3910c9de560dSAlex Tomas /* found anything to free? */ 3911c9de560dSAlex Tomas if (list_empty(&list)) { 3912c9de560dSAlex Tomas BUG_ON(free != 0); 3913c9de560dSAlex Tomas goto out; 3914c9de560dSAlex Tomas } 3915c9de560dSAlex Tomas 3916c9de560dSAlex Tomas /* now free all selected PAs */ 3917c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3918c9de560dSAlex Tomas 3919c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 3920c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3921c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3922c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3923c9de560dSAlex Tomas 3924c9de560dSAlex Tomas if (pa->pa_linear) 3925c9de560dSAlex Tomas ext4_mb_release_group_pa(&e4b, pa); 3926c9de560dSAlex Tomas else 3927c9de560dSAlex Tomas ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 3928c9de560dSAlex Tomas 3929c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3930c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3931c9de560dSAlex Tomas } 3932c9de560dSAlex Tomas 3933c9de560dSAlex Tomas out: 3934c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3935c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 3936c9de560dSAlex Tomas put_bh(bitmap_bh); 3937c9de560dSAlex Tomas return free; 3938c9de560dSAlex Tomas } 3939c9de560dSAlex Tomas 3940c9de560dSAlex Tomas /* 3941c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 3942c9de560dSAlex Tomas * 3943c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 3944c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 3945c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 3946c9de560dSAlex Tomas * 3947c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 3948c9de560dSAlex Tomas */ 3949c9de560dSAlex Tomas void ext4_mb_discard_inode_preallocations(struct inode *inode) 3950c9de560dSAlex Tomas { 3951c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 3952c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 3953c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3954c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3955c9de560dSAlex Tomas ext4_group_t group = 0; 3956c9de560dSAlex Tomas struct list_head list; 3957c9de560dSAlex Tomas struct ext4_buddy e4b; 3958c9de560dSAlex Tomas int err; 3959c9de560dSAlex Tomas 3960c9de560dSAlex Tomas if (!test_opt(sb, MBALLOC) || !S_ISREG(inode->i_mode)) { 3961c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 3962c9de560dSAlex Tomas return; 3963c9de560dSAlex Tomas } 3964c9de560dSAlex Tomas 3965c9de560dSAlex Tomas mb_debug("discard preallocation for inode %lu\n", inode->i_ino); 3966c9de560dSAlex Tomas 3967c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3968c9de560dSAlex Tomas 3969c9de560dSAlex Tomas repeat: 3970c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 3971c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 3972c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 3973c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 3974c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 3975c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 3976c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3977c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3978c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 3979c9de560dSAlex Tomas * use preallocation while we're discarding it */ 3980c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3981c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3982c9de560dSAlex Tomas printk(KERN_ERR "uh-oh! used pa while discarding\n"); 3983c9de560dSAlex Tomas WARN_ON(1); 3984c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 3985c9de560dSAlex Tomas goto repeat; 3986c9de560dSAlex Tomas 3987c9de560dSAlex Tomas } 3988c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3989c9de560dSAlex Tomas pa->pa_deleted = 1; 3990c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3991c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3992c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3993c9de560dSAlex Tomas continue; 3994c9de560dSAlex Tomas } 3995c9de560dSAlex Tomas 3996c9de560dSAlex Tomas /* someone is deleting pa right now */ 3997c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3998c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3999c9de560dSAlex Tomas 4000c9de560dSAlex Tomas /* we have to wait here because pa_deleted 4001c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 4002c9de560dSAlex Tomas * the list. as we might be called from 4003c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 4004c9de560dSAlex Tomas * and concurrent thread which is unlinking 4005c9de560dSAlex Tomas * pa from inode's list may access already 4006c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 4007c9de560dSAlex Tomas 4008c9de560dSAlex Tomas /* XXX: if this happens too often, we can 4009c9de560dSAlex Tomas * add a flag to force wait only in case 4010c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 4011c9de560dSAlex Tomas * regular truncate */ 4012c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 4013c9de560dSAlex Tomas goto repeat; 4014c9de560dSAlex Tomas } 4015c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 4016c9de560dSAlex Tomas 4017c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 4018c9de560dSAlex Tomas BUG_ON(pa->pa_linear != 0); 4019c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL); 4020c9de560dSAlex Tomas 4021c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 4022c9de560dSAlex Tomas BUG_ON(err != 0); /* error handling here */ 4023c9de560dSAlex Tomas 4024c9de560dSAlex Tomas bitmap_bh = read_block_bitmap(sb, group); 4025c9de560dSAlex Tomas if (bitmap_bh == NULL) { 4026c9de560dSAlex Tomas /* error handling here */ 4027c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4028c9de560dSAlex Tomas BUG_ON(bitmap_bh == NULL); 4029c9de560dSAlex Tomas } 4030c9de560dSAlex Tomas 4031c9de560dSAlex Tomas ext4_lock_group(sb, group); 4032c9de560dSAlex Tomas list_del(&pa->pa_group_list); 4033c9de560dSAlex Tomas ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 4034c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4035c9de560dSAlex Tomas 4036c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4037c9de560dSAlex Tomas put_bh(bitmap_bh); 4038c9de560dSAlex Tomas 4039c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 4040c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 4041c9de560dSAlex Tomas } 4042c9de560dSAlex Tomas } 4043c9de560dSAlex Tomas 4044c9de560dSAlex Tomas /* 4045c9de560dSAlex Tomas * finds all preallocated spaces and return blocks being freed to them 4046c9de560dSAlex Tomas * if preallocated space becomes full (no block is used from the space) 4047c9de560dSAlex Tomas * then the function frees space in buddy 4048c9de560dSAlex Tomas * XXX: at the moment, truncate (which is the only way to free blocks) 4049c9de560dSAlex Tomas * discards all preallocations 4050c9de560dSAlex Tomas */ 4051c9de560dSAlex Tomas static void ext4_mb_return_to_preallocation(struct inode *inode, 4052c9de560dSAlex Tomas struct ext4_buddy *e4b, 4053c9de560dSAlex Tomas sector_t block, int count) 4054c9de560dSAlex Tomas { 4055c9de560dSAlex Tomas BUG_ON(!list_empty(&EXT4_I(inode)->i_prealloc_list)); 4056c9de560dSAlex Tomas } 4057c9de560dSAlex Tomas #ifdef MB_DEBUG 4058c9de560dSAlex Tomas static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4059c9de560dSAlex Tomas { 4060c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 4061c9de560dSAlex Tomas ext4_group_t i; 4062c9de560dSAlex Tomas 4063c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: Can't allocate:" 4064c9de560dSAlex Tomas " Allocation context details:\n"); 4065c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: status %d flags %d\n", 4066c9de560dSAlex Tomas ac->ac_status, ac->ac_flags); 4067c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: orig %lu/%lu/%lu@%lu, goal %lu/%lu/%lu@%lu, " 4068c9de560dSAlex Tomas "best %lu/%lu/%lu@%lu cr %d\n", 4069c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_group, 4070c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_start, 4071c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_len, 4072c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_logical, 4073c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_group, 4074c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_start, 4075c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_len, 4076c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_logical, 4077c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_group, 4078c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_start, 4079c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_len, 4080c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_logical, 4081c9de560dSAlex Tomas (int)ac->ac_criteria); 4082c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: %lu scanned, %d found\n", ac->ac_ex_scanned, 4083c9de560dSAlex Tomas ac->ac_found); 4084c9de560dSAlex Tomas printk(KERN_ERR "EXT4-fs: groups: \n"); 4085c9de560dSAlex Tomas for (i = 0; i < EXT4_SB(sb)->s_groups_count; i++) { 4086c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 4087c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 4088c9de560dSAlex Tomas ext4_grpblk_t start; 4089c9de560dSAlex Tomas struct list_head *cur; 4090c9de560dSAlex Tomas ext4_lock_group(sb, i); 4091c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 4092c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 4093c9de560dSAlex Tomas pa_group_list); 4094c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 4095c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 4096c9de560dSAlex Tomas NULL, &start); 4097c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 4098c9de560dSAlex Tomas printk(KERN_ERR "PA:%lu:%d:%u \n", i, 4099c9de560dSAlex Tomas start, pa->pa_len); 4100c9de560dSAlex Tomas } 4101c9de560dSAlex Tomas ext4_lock_group(sb, i); 4102c9de560dSAlex Tomas 4103c9de560dSAlex Tomas if (grp->bb_free == 0) 4104c9de560dSAlex Tomas continue; 4105c9de560dSAlex Tomas printk(KERN_ERR "%lu: %d/%d \n", 4106c9de560dSAlex Tomas i, grp->bb_free, grp->bb_fragments); 4107c9de560dSAlex Tomas } 4108c9de560dSAlex Tomas printk(KERN_ERR "\n"); 4109c9de560dSAlex Tomas } 4110c9de560dSAlex Tomas #else 4111c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 4112c9de560dSAlex Tomas { 4113c9de560dSAlex Tomas return; 4114c9de560dSAlex Tomas } 4115c9de560dSAlex Tomas #endif 4116c9de560dSAlex Tomas 4117c9de560dSAlex Tomas /* 4118c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 4119c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 4120c9de560dSAlex Tomas * allocation which ever is larger 4121c9de560dSAlex Tomas * 4122c9de560dSAlex Tomas * One can tune this size via /proc/fs/ext4/<partition>/stream_req 4123c9de560dSAlex Tomas */ 4124c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 4125c9de560dSAlex Tomas { 4126c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 4127c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 4128c9de560dSAlex Tomas loff_t size, isize; 4129c9de560dSAlex Tomas 4130c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 4131c9de560dSAlex Tomas return; 4132c9de560dSAlex Tomas 4133c9de560dSAlex Tomas size = ac->ac_o_ex.fe_logical + ac->ac_o_ex.fe_len; 4134c9de560dSAlex Tomas isize = i_size_read(ac->ac_inode) >> bsbits; 4135c9de560dSAlex Tomas size = max(size, isize); 4136c9de560dSAlex Tomas 4137c9de560dSAlex Tomas /* don't use group allocation for large files */ 4138c9de560dSAlex Tomas if (size >= sbi->s_mb_stream_request) 4139c9de560dSAlex Tomas return; 4140c9de560dSAlex Tomas 4141c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 4142c9de560dSAlex Tomas return; 4143c9de560dSAlex Tomas 4144c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 4145c9de560dSAlex Tomas /* 4146c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 4147c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 4148c9de560dSAlex Tomas * request from multiple CPUs. 4149c9de560dSAlex Tomas */ 4150c9de560dSAlex Tomas ac->ac_lg = &sbi->s_locality_groups[get_cpu()]; 4151c9de560dSAlex Tomas put_cpu(); 4152c9de560dSAlex Tomas 4153c9de560dSAlex Tomas /* we're going to use group allocation */ 4154c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4155c9de560dSAlex Tomas 4156c9de560dSAlex Tomas /* serialize all allocations in the group */ 4157c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4158c9de560dSAlex Tomas } 4159c9de560dSAlex Tomas 4160c9de560dSAlex Tomas static int ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4161c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4162c9de560dSAlex Tomas { 4163c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4164c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4165c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4166c9de560dSAlex Tomas ext4_group_t group; 4167c9de560dSAlex Tomas unsigned long len; 4168c9de560dSAlex Tomas unsigned long goal; 4169c9de560dSAlex Tomas ext4_grpblk_t block; 4170c9de560dSAlex Tomas 4171c9de560dSAlex Tomas /* we can't allocate > group size */ 4172c9de560dSAlex Tomas len = ar->len; 4173c9de560dSAlex Tomas 4174c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 4175c9de560dSAlex Tomas if (len >= EXT4_BLOCKS_PER_GROUP(sb) - 10) 4176c9de560dSAlex Tomas len = EXT4_BLOCKS_PER_GROUP(sb) - 10; 4177c9de560dSAlex Tomas 4178c9de560dSAlex Tomas /* start searching from the goal */ 4179c9de560dSAlex Tomas goal = ar->goal; 4180c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4181c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4182c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4183c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4184c9de560dSAlex Tomas 4185c9de560dSAlex Tomas /* set up allocation goals */ 4186c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ar->logical; 4187c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 4188c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 4189c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 4190c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4191c9de560dSAlex Tomas ac->ac_groups_scanned = 0; 4192c9de560dSAlex Tomas ac->ac_ex_scanned = 0; 4193c9de560dSAlex Tomas ac->ac_found = 0; 4194c9de560dSAlex Tomas ac->ac_sb = sb; 4195c9de560dSAlex Tomas ac->ac_inode = ar->inode; 4196c9de560dSAlex Tomas ac->ac_o_ex.fe_logical = ar->logical; 4197c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4198c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4199c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 4200c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = ar->logical; 4201c9de560dSAlex Tomas ac->ac_g_ex.fe_group = group; 4202c9de560dSAlex Tomas ac->ac_g_ex.fe_start = block; 4203c9de560dSAlex Tomas ac->ac_g_ex.fe_len = len; 4204c9de560dSAlex Tomas ac->ac_f_ex.fe_len = 0; 4205c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4206c9de560dSAlex Tomas ac->ac_2order = 0; 4207c9de560dSAlex Tomas ac->ac_criteria = 0; 4208c9de560dSAlex Tomas ac->ac_pa = NULL; 4209c9de560dSAlex Tomas ac->ac_bitmap_page = NULL; 4210c9de560dSAlex Tomas ac->ac_buddy_page = NULL; 4211c9de560dSAlex Tomas ac->ac_lg = NULL; 4212c9de560dSAlex Tomas 4213c9de560dSAlex Tomas /* we have to define context: we'll we work with a file or 4214c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4215c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4216c9de560dSAlex Tomas 4217c9de560dSAlex Tomas mb_debug("init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, " 4218c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4219c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4220c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4221c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4222c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 4223c9de560dSAlex Tomas atomic_read(&ar->inode->i_writecount) ? "" : "non-"); 4224c9de560dSAlex Tomas return 0; 4225c9de560dSAlex Tomas 4226c9de560dSAlex Tomas } 4227c9de560dSAlex Tomas 4228c9de560dSAlex Tomas /* 4229c9de560dSAlex Tomas * release all resource we used in allocation 4230c9de560dSAlex Tomas */ 4231c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4232c9de560dSAlex Tomas { 4233c9de560dSAlex Tomas if (ac->ac_pa) { 4234c9de560dSAlex Tomas if (ac->ac_pa->pa_linear) { 4235c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 4236c9de560dSAlex Tomas spin_lock(&ac->ac_pa->pa_lock); 4237c9de560dSAlex Tomas ac->ac_pa->pa_pstart += ac->ac_b_ex.fe_len; 4238c9de560dSAlex Tomas ac->ac_pa->pa_lstart += ac->ac_b_ex.fe_len; 4239c9de560dSAlex Tomas ac->ac_pa->pa_free -= ac->ac_b_ex.fe_len; 4240c9de560dSAlex Tomas ac->ac_pa->pa_len -= ac->ac_b_ex.fe_len; 4241c9de560dSAlex Tomas spin_unlock(&ac->ac_pa->pa_lock); 4242c9de560dSAlex Tomas } 4243c9de560dSAlex Tomas ext4_mb_put_pa(ac, ac->ac_sb, ac->ac_pa); 4244c9de560dSAlex Tomas } 4245c9de560dSAlex Tomas if (ac->ac_bitmap_page) 4246c9de560dSAlex Tomas page_cache_release(ac->ac_bitmap_page); 4247c9de560dSAlex Tomas if (ac->ac_buddy_page) 4248c9de560dSAlex Tomas page_cache_release(ac->ac_buddy_page); 4249c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4250c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4251c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4252c9de560dSAlex Tomas return 0; 4253c9de560dSAlex Tomas } 4254c9de560dSAlex Tomas 4255c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4256c9de560dSAlex Tomas { 4257c9de560dSAlex Tomas ext4_group_t i; 4258c9de560dSAlex Tomas int ret; 4259c9de560dSAlex Tomas int freed = 0; 4260c9de560dSAlex Tomas 4261c9de560dSAlex Tomas for (i = 0; i < EXT4_SB(sb)->s_groups_count && needed > 0; i++) { 4262c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4263c9de560dSAlex Tomas freed += ret; 4264c9de560dSAlex Tomas needed -= ret; 4265c9de560dSAlex Tomas } 4266c9de560dSAlex Tomas 4267c9de560dSAlex Tomas return freed; 4268c9de560dSAlex Tomas } 4269c9de560dSAlex Tomas 4270c9de560dSAlex Tomas /* 4271c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4272c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4273c9de560dSAlex Tomas * to usual allocation 4274c9de560dSAlex Tomas */ 4275c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4276c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4277c9de560dSAlex Tomas { 4278256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4279c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4280c9de560dSAlex Tomas struct super_block *sb; 4281c9de560dSAlex Tomas ext4_fsblk_t block = 0; 4282c9de560dSAlex Tomas int freed; 4283c9de560dSAlex Tomas int inquota; 4284c9de560dSAlex Tomas 4285c9de560dSAlex Tomas sb = ar->inode->i_sb; 4286c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4287c9de560dSAlex Tomas 4288c9de560dSAlex Tomas if (!test_opt(sb, MBALLOC)) { 4289c9de560dSAlex Tomas block = ext4_new_blocks_old(handle, ar->inode, ar->goal, 4290c9de560dSAlex Tomas &(ar->len), errp); 4291c9de560dSAlex Tomas return block; 4292c9de560dSAlex Tomas } 4293c9de560dSAlex Tomas 4294c9de560dSAlex Tomas while (ar->len && DQUOT_ALLOC_BLOCK(ar->inode, ar->len)) { 4295c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4296c9de560dSAlex Tomas ar->len--; 4297c9de560dSAlex Tomas } 4298c9de560dSAlex Tomas if (ar->len == 0) { 4299c9de560dSAlex Tomas *errp = -EDQUOT; 4300c9de560dSAlex Tomas return 0; 4301c9de560dSAlex Tomas } 4302c9de560dSAlex Tomas inquota = ar->len; 4303c9de560dSAlex Tomas 4304256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 4305256bdb49SEric Sandeen if (!ac) { 4306256bdb49SEric Sandeen *errp = -ENOMEM; 4307256bdb49SEric Sandeen return 0; 4308256bdb49SEric Sandeen } 4309256bdb49SEric Sandeen 4310c9de560dSAlex Tomas ext4_mb_poll_new_transaction(sb, handle); 4311c9de560dSAlex Tomas 4312256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4313c9de560dSAlex Tomas if (*errp) { 4314c9de560dSAlex Tomas ar->len = 0; 4315c9de560dSAlex Tomas goto out; 4316c9de560dSAlex Tomas } 4317c9de560dSAlex Tomas 4318256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 4319256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4320c9de560dSAlex Tomas 4321256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4322256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 4323c9de560dSAlex Tomas 4324c9de560dSAlex Tomas repeat: 4325c9de560dSAlex Tomas /* allocate space in core */ 4326256bdb49SEric Sandeen ext4_mb_regular_allocator(ac); 4327c9de560dSAlex Tomas 4328c9de560dSAlex Tomas /* as we've just preallocated more space than 4329c9de560dSAlex Tomas * user requested orinally, we store allocated 4330c9de560dSAlex Tomas * space in a special descriptor */ 4331256bdb49SEric Sandeen if (ac->ac_status == AC_STATUS_FOUND && 4332256bdb49SEric Sandeen ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 4333256bdb49SEric Sandeen ext4_mb_new_preallocation(ac); 4334c9de560dSAlex Tomas } 4335c9de560dSAlex Tomas 4336256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 4337256bdb49SEric Sandeen ext4_mb_mark_diskspace_used(ac, handle); 4338c9de560dSAlex Tomas *errp = 0; 4339256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4340256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4341c9de560dSAlex Tomas } else { 4342256bdb49SEric Sandeen freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 4343c9de560dSAlex Tomas if (freed) 4344c9de560dSAlex Tomas goto repeat; 4345c9de560dSAlex Tomas *errp = -ENOSPC; 4346256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4347c9de560dSAlex Tomas ar->len = 0; 4348256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4349c9de560dSAlex Tomas } 4350c9de560dSAlex Tomas 4351256bdb49SEric Sandeen ext4_mb_release_context(ac); 4352c9de560dSAlex Tomas 4353c9de560dSAlex Tomas out: 4354c9de560dSAlex Tomas if (ar->len < inquota) 4355c9de560dSAlex Tomas DQUOT_FREE_BLOCK(ar->inode, inquota - ar->len); 4356c9de560dSAlex Tomas 4357256bdb49SEric Sandeen kmem_cache_free(ext4_ac_cachep, ac); 4358c9de560dSAlex Tomas return block; 4359c9de560dSAlex Tomas } 4360c9de560dSAlex Tomas static void ext4_mb_poll_new_transaction(struct super_block *sb, 4361c9de560dSAlex Tomas handle_t *handle) 4362c9de560dSAlex Tomas { 4363c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4364c9de560dSAlex Tomas 4365c9de560dSAlex Tomas if (sbi->s_last_transaction == handle->h_transaction->t_tid) 4366c9de560dSAlex Tomas return; 4367c9de560dSAlex Tomas 4368c9de560dSAlex Tomas /* new transaction! time to close last one and free blocks for 4369c9de560dSAlex Tomas * committed transaction. we know that only transaction can be 4370c9de560dSAlex Tomas * active, so previos transaction can be being logged and we 4371c9de560dSAlex Tomas * know that transaction before previous is known to be already 4372c9de560dSAlex Tomas * logged. this means that now we may free blocks freed in all 4373c9de560dSAlex Tomas * transactions before previous one. hope I'm clear enough ... */ 4374c9de560dSAlex Tomas 4375c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 4376c9de560dSAlex Tomas if (sbi->s_last_transaction != handle->h_transaction->t_tid) { 4377c9de560dSAlex Tomas mb_debug("new transaction %lu, old %lu\n", 4378c9de560dSAlex Tomas (unsigned long) handle->h_transaction->t_tid, 4379c9de560dSAlex Tomas (unsigned long) sbi->s_last_transaction); 4380c9de560dSAlex Tomas list_splice_init(&sbi->s_closed_transaction, 4381c9de560dSAlex Tomas &sbi->s_committed_transaction); 4382c9de560dSAlex Tomas list_splice_init(&sbi->s_active_transaction, 4383c9de560dSAlex Tomas &sbi->s_closed_transaction); 4384c9de560dSAlex Tomas sbi->s_last_transaction = handle->h_transaction->t_tid; 4385c9de560dSAlex Tomas } 4386c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 4387c9de560dSAlex Tomas 4388c9de560dSAlex Tomas ext4_mb_free_committed_blocks(sb); 4389c9de560dSAlex Tomas } 4390c9de560dSAlex Tomas 4391c9de560dSAlex Tomas static int ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 4392c9de560dSAlex Tomas ext4_group_t group, ext4_grpblk_t block, int count) 4393c9de560dSAlex Tomas { 4394c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4395c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4396c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4397c9de560dSAlex Tomas struct ext4_free_metadata *md; 4398c9de560dSAlex Tomas int i; 4399c9de560dSAlex Tomas 4400c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4401c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4402c9de560dSAlex Tomas 4403c9de560dSAlex Tomas ext4_lock_group(sb, group); 4404c9de560dSAlex Tomas for (i = 0; i < count; i++) { 4405c9de560dSAlex Tomas md = db->bb_md_cur; 4406c9de560dSAlex Tomas if (md && db->bb_tid != handle->h_transaction->t_tid) { 4407c9de560dSAlex Tomas db->bb_md_cur = NULL; 4408c9de560dSAlex Tomas md = NULL; 4409c9de560dSAlex Tomas } 4410c9de560dSAlex Tomas 4411c9de560dSAlex Tomas if (md == NULL) { 4412c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4413c9de560dSAlex Tomas md = kmalloc(sizeof(*md), GFP_NOFS); 4414c9de560dSAlex Tomas if (md == NULL) 4415c9de560dSAlex Tomas return -ENOMEM; 4416c9de560dSAlex Tomas md->num = 0; 4417c9de560dSAlex Tomas md->group = group; 4418c9de560dSAlex Tomas 4419c9de560dSAlex Tomas ext4_lock_group(sb, group); 4420c9de560dSAlex Tomas if (db->bb_md_cur == NULL) { 4421c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 4422c9de560dSAlex Tomas list_add(&md->list, &sbi->s_active_transaction); 4423c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 4424c9de560dSAlex Tomas /* protect buddy cache from being freed, 4425c9de560dSAlex Tomas * otherwise we'll refresh it from 4426c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4427c9de560dSAlex Tomas * blocks */ 4428c9de560dSAlex Tomas page_cache_get(e4b->bd_buddy_page); 4429c9de560dSAlex Tomas page_cache_get(e4b->bd_bitmap_page); 4430c9de560dSAlex Tomas db->bb_md_cur = md; 4431c9de560dSAlex Tomas db->bb_tid = handle->h_transaction->t_tid; 4432c9de560dSAlex Tomas mb_debug("new md 0x%p for group %lu\n", 4433c9de560dSAlex Tomas md, md->group); 4434c9de560dSAlex Tomas } else { 4435c9de560dSAlex Tomas kfree(md); 4436c9de560dSAlex Tomas md = db->bb_md_cur; 4437c9de560dSAlex Tomas } 4438c9de560dSAlex Tomas } 4439c9de560dSAlex Tomas 4440c9de560dSAlex Tomas BUG_ON(md->num >= EXT4_BB_MAX_BLOCKS); 4441c9de560dSAlex Tomas md->blocks[md->num] = block + i; 4442c9de560dSAlex Tomas md->num++; 4443c9de560dSAlex Tomas if (md->num == EXT4_BB_MAX_BLOCKS) { 4444c9de560dSAlex Tomas /* no more space, put full container on a sb's list */ 4445c9de560dSAlex Tomas db->bb_md_cur = NULL; 4446c9de560dSAlex Tomas } 4447c9de560dSAlex Tomas } 4448c9de560dSAlex Tomas ext4_unlock_group(sb, group); 4449c9de560dSAlex Tomas return 0; 4450c9de560dSAlex Tomas } 4451c9de560dSAlex Tomas 4452c9de560dSAlex Tomas /* 4453c9de560dSAlex Tomas * Main entry point into mballoc to free blocks 4454c9de560dSAlex Tomas */ 4455c9de560dSAlex Tomas void ext4_mb_free_blocks(handle_t *handle, struct inode *inode, 4456c9de560dSAlex Tomas unsigned long block, unsigned long count, 4457c9de560dSAlex Tomas int metadata, unsigned long *freed) 4458c9de560dSAlex Tomas { 445926346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 4460c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4461256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4462c9de560dSAlex Tomas struct ext4_group_desc *gdp; 4463c9de560dSAlex Tomas struct ext4_super_block *es; 4464c9de560dSAlex Tomas unsigned long overflow; 4465c9de560dSAlex Tomas ext4_grpblk_t bit; 4466c9de560dSAlex Tomas struct buffer_head *gd_bh; 4467c9de560dSAlex Tomas ext4_group_t block_group; 4468c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4469c9de560dSAlex Tomas struct ext4_buddy e4b; 4470c9de560dSAlex Tomas int err = 0; 4471c9de560dSAlex Tomas int ret; 4472c9de560dSAlex Tomas 4473c9de560dSAlex Tomas *freed = 0; 4474c9de560dSAlex Tomas 4475c9de560dSAlex Tomas ext4_mb_poll_new_transaction(sb, handle); 4476c9de560dSAlex Tomas 4477c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4478c9de560dSAlex Tomas es = EXT4_SB(sb)->s_es; 4479c9de560dSAlex Tomas if (block < le32_to_cpu(es->s_first_data_block) || 4480c9de560dSAlex Tomas block + count < block || 4481c9de560dSAlex Tomas block + count > ext4_blocks_count(es)) { 4482c9de560dSAlex Tomas ext4_error(sb, __FUNCTION__, 4483c9de560dSAlex Tomas "Freeing blocks not in datazone - " 4484c9de560dSAlex Tomas "block = %lu, count = %lu", block, count); 4485c9de560dSAlex Tomas goto error_return; 4486c9de560dSAlex Tomas } 4487c9de560dSAlex Tomas 4488c9de560dSAlex Tomas ext4_debug("freeing block %lu\n", block); 4489c9de560dSAlex Tomas 4490256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 4491256bdb49SEric Sandeen if (ac) { 4492256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_FREE; 4493256bdb49SEric Sandeen ac->ac_inode = inode; 4494256bdb49SEric Sandeen ac->ac_sb = sb; 4495256bdb49SEric Sandeen } 4496c9de560dSAlex Tomas 4497c9de560dSAlex Tomas do_more: 4498c9de560dSAlex Tomas overflow = 0; 4499c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 4500c9de560dSAlex Tomas 4501c9de560dSAlex Tomas /* 4502c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 4503c9de560dSAlex Tomas * boundary. 4504c9de560dSAlex Tomas */ 4505c9de560dSAlex Tomas if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) { 4506c9de560dSAlex Tomas overflow = bit + count - EXT4_BLOCKS_PER_GROUP(sb); 4507c9de560dSAlex Tomas count -= overflow; 4508c9de560dSAlex Tomas } 4509c9de560dSAlex Tomas bitmap_bh = read_block_bitmap(sb, block_group); 4510c9de560dSAlex Tomas if (!bitmap_bh) 4511c9de560dSAlex Tomas goto error_return; 4512c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 4513c9de560dSAlex Tomas if (!gdp) 4514c9de560dSAlex Tomas goto error_return; 4515c9de560dSAlex Tomas 4516c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 4517c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 4518c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 4519c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group) || 4520c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 4521c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group)) { 4522c9de560dSAlex Tomas 4523c9de560dSAlex Tomas ext4_error(sb, __FUNCTION__, 4524c9de560dSAlex Tomas "Freeing blocks in system zone - " 4525c9de560dSAlex Tomas "Block = %lu, count = %lu", block, count); 4526c9de560dSAlex Tomas } 4527c9de560dSAlex Tomas 4528c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 4529c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 4530c9de560dSAlex Tomas if (err) 4531c9de560dSAlex Tomas goto error_return; 4532c9de560dSAlex Tomas 4533c9de560dSAlex Tomas /* 4534c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 4535c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 4536c9de560dSAlex Tomas * using it 4537c9de560dSAlex Tomas */ 4538c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 4539c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 4540c9de560dSAlex Tomas if (err) 4541c9de560dSAlex Tomas goto error_return; 4542c9de560dSAlex Tomas 4543c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, block_group, &e4b); 4544c9de560dSAlex Tomas if (err) 4545c9de560dSAlex Tomas goto error_return; 4546c9de560dSAlex Tomas 4547c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4548c9de560dSAlex Tomas { 4549c9de560dSAlex Tomas int i; 4550c9de560dSAlex Tomas for (i = 0; i < count; i++) 4551c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 4552c9de560dSAlex Tomas } 4553c9de560dSAlex Tomas #endif 4554c9de560dSAlex Tomas mb_clear_bits(sb_bgl_lock(sbi, block_group), bitmap_bh->b_data, 4555c9de560dSAlex Tomas bit, count); 4556c9de560dSAlex Tomas 4557c9de560dSAlex Tomas /* We dirtied the bitmap block */ 4558c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 4559c9de560dSAlex Tomas err = ext4_journal_dirty_metadata(handle, bitmap_bh); 4560c9de560dSAlex Tomas 4561256bdb49SEric Sandeen if (ac) { 4562256bdb49SEric Sandeen ac->ac_b_ex.fe_group = block_group; 4563256bdb49SEric Sandeen ac->ac_b_ex.fe_start = bit; 4564256bdb49SEric Sandeen ac->ac_b_ex.fe_len = count; 4565256bdb49SEric Sandeen ext4_mb_store_history(ac); 4566256bdb49SEric Sandeen } 4567c9de560dSAlex Tomas 4568c9de560dSAlex Tomas if (metadata) { 4569c9de560dSAlex Tomas /* blocks being freed are metadata. these blocks shouldn't 4570c9de560dSAlex Tomas * be used until this transaction is committed */ 4571c9de560dSAlex Tomas ext4_mb_free_metadata(handle, &e4b, block_group, bit, count); 4572c9de560dSAlex Tomas } else { 4573c9de560dSAlex Tomas ext4_lock_group(sb, block_group); 4574c9de560dSAlex Tomas err = mb_free_blocks(inode, &e4b, bit, count); 4575c9de560dSAlex Tomas ext4_mb_return_to_preallocation(inode, &e4b, block, count); 4576c9de560dSAlex Tomas ext4_unlock_group(sb, block_group); 4577c9de560dSAlex Tomas BUG_ON(err != 0); 4578c9de560dSAlex Tomas } 4579c9de560dSAlex Tomas 4580c9de560dSAlex Tomas spin_lock(sb_bgl_lock(sbi, block_group)); 4581c9de560dSAlex Tomas gdp->bg_free_blocks_count = 4582c9de560dSAlex Tomas cpu_to_le16(le16_to_cpu(gdp->bg_free_blocks_count) + count); 4583c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp); 4584c9de560dSAlex Tomas spin_unlock(sb_bgl_lock(sbi, block_group)); 4585c9de560dSAlex Tomas percpu_counter_add(&sbi->s_freeblocks_counter, count); 4586c9de560dSAlex Tomas 4587c9de560dSAlex Tomas ext4_mb_release_desc(&e4b); 4588c9de560dSAlex Tomas 4589c9de560dSAlex Tomas *freed += count; 4590c9de560dSAlex Tomas 4591c9de560dSAlex Tomas /* And the group descriptor block */ 4592c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 4593c9de560dSAlex Tomas ret = ext4_journal_dirty_metadata(handle, gd_bh); 4594c9de560dSAlex Tomas if (!err) 4595c9de560dSAlex Tomas err = ret; 4596c9de560dSAlex Tomas 4597c9de560dSAlex Tomas if (overflow && !err) { 4598c9de560dSAlex Tomas block += count; 4599c9de560dSAlex Tomas count = overflow; 4600c9de560dSAlex Tomas put_bh(bitmap_bh); 4601c9de560dSAlex Tomas goto do_more; 4602c9de560dSAlex Tomas } 4603c9de560dSAlex Tomas sb->s_dirt = 1; 4604c9de560dSAlex Tomas error_return: 4605c9de560dSAlex Tomas brelse(bitmap_bh); 4606c9de560dSAlex Tomas ext4_std_error(sb, err); 4607256bdb49SEric Sandeen if (ac) 4608256bdb49SEric Sandeen kmem_cache_free(ext4_ac_cachep, ac); 4609c9de560dSAlex Tomas return; 4610c9de560dSAlex Tomas } 4611