11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * 2002-10-18 written by Jim Houston jim.houston@ccur.com 31da177e4SLinus Torvalds * Copyright (C) 2002 by Concurrent Computer Corporation 41da177e4SLinus Torvalds * Distributed under the GNU GPL license version 2. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Modified by George Anzinger to reuse immediately and to use 71da177e4SLinus Torvalds * find bit instructions. Also removed _irq on spinlocks. 81da177e4SLinus Torvalds * 93219b3b7SNadia Derbey * Modified by Nadia Derbey to make it RCU safe. 103219b3b7SNadia Derbey * 111da177e4SLinus Torvalds * Small id to pointer translation service. 121da177e4SLinus Torvalds * 131da177e4SLinus Torvalds * It uses a radix tree like structure as a sparse array indexed 141da177e4SLinus Torvalds * by the id to obtain the pointer. The bitmap makes allocating 151da177e4SLinus Torvalds * a new id quick. 161da177e4SLinus Torvalds * 171da177e4SLinus Torvalds * You call it to allocate an id (an int) an associate with that id a 181da177e4SLinus Torvalds * pointer or what ever, we treat it as a (void *). You can pass this 191da177e4SLinus Torvalds * id to a user for him to pass back at a later time. You then pass 201da177e4SLinus Torvalds * that id to this code and it returns your pointer. 211da177e4SLinus Torvalds 221da177e4SLinus Torvalds * You can release ids at any time. When all ids are released, most of 23125c4c70SFengguang Wu * the memory is returned (we keep MAX_IDR_FREE) in a local pool so we 241da177e4SLinus Torvalds * don't need to go to the memory "store" during an id allocate, just 251da177e4SLinus Torvalds * so you don't need to be too concerned about locking and conflicts 261da177e4SLinus Torvalds * with the slab allocator. 271da177e4SLinus Torvalds */ 281da177e4SLinus Torvalds 291da177e4SLinus Torvalds #ifndef TEST // to test in user space... 301da177e4SLinus Torvalds #include <linux/slab.h> 311da177e4SLinus Torvalds #include <linux/init.h> 328bc3bcc9SPaul Gortmaker #include <linux/export.h> 331da177e4SLinus Torvalds #endif 345806f07cSJeff Mahoney #include <linux/err.h> 351da177e4SLinus Torvalds #include <linux/string.h> 361da177e4SLinus Torvalds #include <linux/idr.h> 3788eca020SRusty Russell #include <linux/spinlock.h> 381da177e4SLinus Torvalds 39e18b890bSChristoph Lameter static struct kmem_cache *idr_layer_cache; 4088eca020SRusty Russell static DEFINE_SPINLOCK(simple_ida_lock); 411da177e4SLinus Torvalds 424ae53789SNadia Derbey static struct idr_layer *get_from_free_list(struct idr *idp) 431da177e4SLinus Torvalds { 441da177e4SLinus Torvalds struct idr_layer *p; 45c259cc28SRoland Dreier unsigned long flags; 461da177e4SLinus Torvalds 47c259cc28SRoland Dreier spin_lock_irqsave(&idp->lock, flags); 481da177e4SLinus Torvalds if ((p = idp->id_free)) { 491da177e4SLinus Torvalds idp->id_free = p->ary[0]; 501da177e4SLinus Torvalds idp->id_free_cnt--; 511da177e4SLinus Torvalds p->ary[0] = NULL; 521da177e4SLinus Torvalds } 53c259cc28SRoland Dreier spin_unlock_irqrestore(&idp->lock, flags); 541da177e4SLinus Torvalds return(p); 551da177e4SLinus Torvalds } 561da177e4SLinus Torvalds 57cf481c20SNadia Derbey static void idr_layer_rcu_free(struct rcu_head *head) 58cf481c20SNadia Derbey { 59cf481c20SNadia Derbey struct idr_layer *layer; 60cf481c20SNadia Derbey 61cf481c20SNadia Derbey layer = container_of(head, struct idr_layer, rcu_head); 62cf481c20SNadia Derbey kmem_cache_free(idr_layer_cache, layer); 63cf481c20SNadia Derbey } 64cf481c20SNadia Derbey 65cf481c20SNadia Derbey static inline void free_layer(struct idr_layer *p) 66cf481c20SNadia Derbey { 67cf481c20SNadia Derbey call_rcu(&p->rcu_head, idr_layer_rcu_free); 68cf481c20SNadia Derbey } 69cf481c20SNadia Derbey 701eec0056SSonny Rao /* only called when idp->lock is held */ 714ae53789SNadia Derbey static void __move_to_free_list(struct idr *idp, struct idr_layer *p) 721eec0056SSonny Rao { 731eec0056SSonny Rao p->ary[0] = idp->id_free; 741eec0056SSonny Rao idp->id_free = p; 751eec0056SSonny Rao idp->id_free_cnt++; 761eec0056SSonny Rao } 771eec0056SSonny Rao 784ae53789SNadia Derbey static void move_to_free_list(struct idr *idp, struct idr_layer *p) 791da177e4SLinus Torvalds { 80c259cc28SRoland Dreier unsigned long flags; 81c259cc28SRoland Dreier 821da177e4SLinus Torvalds /* 831da177e4SLinus Torvalds * Depends on the return element being zeroed. 841da177e4SLinus Torvalds */ 85c259cc28SRoland Dreier spin_lock_irqsave(&idp->lock, flags); 864ae53789SNadia Derbey __move_to_free_list(idp, p); 87c259cc28SRoland Dreier spin_unlock_irqrestore(&idp->lock, flags); 881da177e4SLinus Torvalds } 891da177e4SLinus Torvalds 90e33ac8bdSTejun Heo static void idr_mark_full(struct idr_layer **pa, int id) 91e33ac8bdSTejun Heo { 92e33ac8bdSTejun Heo struct idr_layer *p = pa[0]; 93e33ac8bdSTejun Heo int l = 0; 94e33ac8bdSTejun Heo 95e33ac8bdSTejun Heo __set_bit(id & IDR_MASK, &p->bitmap); 96e33ac8bdSTejun Heo /* 97e33ac8bdSTejun Heo * If this layer is full mark the bit in the layer above to 98e33ac8bdSTejun Heo * show that this part of the radix tree is full. This may 99e33ac8bdSTejun Heo * complete the layer above and require walking up the radix 100e33ac8bdSTejun Heo * tree. 101e33ac8bdSTejun Heo */ 102e33ac8bdSTejun Heo while (p->bitmap == IDR_FULL) { 103e33ac8bdSTejun Heo if (!(p = pa[++l])) 104e33ac8bdSTejun Heo break; 105e33ac8bdSTejun Heo id = id >> IDR_BITS; 106e33ac8bdSTejun Heo __set_bit((id & IDR_MASK), &p->bitmap); 107e33ac8bdSTejun Heo } 108e33ac8bdSTejun Heo } 109e33ac8bdSTejun Heo 1101da177e4SLinus Torvalds /** 11156083ab1SRandy Dunlap * idr_pre_get - reserve resources for idr allocation 1121da177e4SLinus Torvalds * @idp: idr handle 1131da177e4SLinus Torvalds * @gfp_mask: memory allocation flags 1141da177e4SLinus Torvalds * 115066a9be6SNaohiro Aota * This function should be called prior to calling the idr_get_new* functions. 116066a9be6SNaohiro Aota * It preallocates enough memory to satisfy the worst possible allocation. The 117066a9be6SNaohiro Aota * caller should pass in GFP_KERNEL if possible. This of course requires that 118066a9be6SNaohiro Aota * no spinning locks be held. 1191da177e4SLinus Torvalds * 12056083ab1SRandy Dunlap * If the system is REALLY out of memory this function returns %0, 12156083ab1SRandy Dunlap * otherwise %1. 1221da177e4SLinus Torvalds */ 123fd4f2df2SAl Viro int idr_pre_get(struct idr *idp, gfp_t gfp_mask) 1241da177e4SLinus Torvalds { 125125c4c70SFengguang Wu while (idp->id_free_cnt < MAX_IDR_FREE) { 1261da177e4SLinus Torvalds struct idr_layer *new; 1275b019e99SAndrew Morton new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); 1281da177e4SLinus Torvalds if (new == NULL) 1291da177e4SLinus Torvalds return (0); 1304ae53789SNadia Derbey move_to_free_list(idp, new); 1311da177e4SLinus Torvalds } 1321da177e4SLinus Torvalds return 1; 1331da177e4SLinus Torvalds } 1341da177e4SLinus Torvalds EXPORT_SYMBOL(idr_pre_get); 1351da177e4SLinus Torvalds 136e33ac8bdSTejun Heo static int sub_alloc(struct idr *idp, int *starting_id, struct idr_layer **pa) 1371da177e4SLinus Torvalds { 1381da177e4SLinus Torvalds int n, m, sh; 1391da177e4SLinus Torvalds struct idr_layer *p, *new; 1407aae6dd8STejun Heo int l, id, oid; 1415ba25331SAl Viro unsigned long bm; 1421da177e4SLinus Torvalds 1431da177e4SLinus Torvalds id = *starting_id; 1447aae6dd8STejun Heo restart: 1451da177e4SLinus Torvalds p = idp->top; 1461da177e4SLinus Torvalds l = idp->layers; 1471da177e4SLinus Torvalds pa[l--] = NULL; 1481da177e4SLinus Torvalds while (1) { 1491da177e4SLinus Torvalds /* 1501da177e4SLinus Torvalds * We run around this while until we reach the leaf node... 1511da177e4SLinus Torvalds */ 1521da177e4SLinus Torvalds n = (id >> (IDR_BITS*l)) & IDR_MASK; 1531da177e4SLinus Torvalds bm = ~p->bitmap; 1541da177e4SLinus Torvalds m = find_next_bit(&bm, IDR_SIZE, n); 1551da177e4SLinus Torvalds if (m == IDR_SIZE) { 1561da177e4SLinus Torvalds /* no space available go back to previous layer. */ 1571da177e4SLinus Torvalds l++; 1587aae6dd8STejun Heo oid = id; 1591da177e4SLinus Torvalds id = (id | ((1 << (IDR_BITS * l)) - 1)) + 1; 1607aae6dd8STejun Heo 1617aae6dd8STejun Heo /* if already at the top layer, we need to grow */ 162d2e7276bSTejun Heo if (id >= 1 << (idp->layers * IDR_BITS)) { 1631da177e4SLinus Torvalds *starting_id = id; 164944ca05cSNadia Derbey return IDR_NEED_TO_GROW; 1651da177e4SLinus Torvalds } 166d2e7276bSTejun Heo p = pa[l]; 167d2e7276bSTejun Heo BUG_ON(!p); 1687aae6dd8STejun Heo 1697aae6dd8STejun Heo /* If we need to go up one layer, continue the 1707aae6dd8STejun Heo * loop; otherwise, restart from the top. 1717aae6dd8STejun Heo */ 1727aae6dd8STejun Heo sh = IDR_BITS * (l + 1); 1737aae6dd8STejun Heo if (oid >> sh == id >> sh) 1741da177e4SLinus Torvalds continue; 1757aae6dd8STejun Heo else 1767aae6dd8STejun Heo goto restart; 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds if (m != n) { 1791da177e4SLinus Torvalds sh = IDR_BITS*l; 1801da177e4SLinus Torvalds id = ((id >> sh) ^ n ^ m) << sh; 1811da177e4SLinus Torvalds } 182125c4c70SFengguang Wu if ((id >= MAX_IDR_BIT) || (id < 0)) 183944ca05cSNadia Derbey return IDR_NOMORE_SPACE; 1841da177e4SLinus Torvalds if (l == 0) 1851da177e4SLinus Torvalds break; 1861da177e4SLinus Torvalds /* 1871da177e4SLinus Torvalds * Create the layer below if it is missing. 1881da177e4SLinus Torvalds */ 1891da177e4SLinus Torvalds if (!p->ary[m]) { 1904ae53789SNadia Derbey new = get_from_free_list(idp); 1914ae53789SNadia Derbey if (!new) 1921da177e4SLinus Torvalds return -1; 1936ff2d39bSManfred Spraul new->layer = l-1; 1943219b3b7SNadia Derbey rcu_assign_pointer(p->ary[m], new); 1951da177e4SLinus Torvalds p->count++; 1961da177e4SLinus Torvalds } 1971da177e4SLinus Torvalds pa[l--] = p; 1981da177e4SLinus Torvalds p = p->ary[m]; 1991da177e4SLinus Torvalds } 200e33ac8bdSTejun Heo 201e33ac8bdSTejun Heo pa[l] = p; 202e33ac8bdSTejun Heo return id; 2031da177e4SLinus Torvalds } 2041da177e4SLinus Torvalds 205e33ac8bdSTejun Heo static int idr_get_empty_slot(struct idr *idp, int starting_id, 206e33ac8bdSTejun Heo struct idr_layer **pa) 2071da177e4SLinus Torvalds { 2081da177e4SLinus Torvalds struct idr_layer *p, *new; 2091da177e4SLinus Torvalds int layers, v, id; 210c259cc28SRoland Dreier unsigned long flags; 2111da177e4SLinus Torvalds 2121da177e4SLinus Torvalds id = starting_id; 2131da177e4SLinus Torvalds build_up: 2141da177e4SLinus Torvalds p = idp->top; 2151da177e4SLinus Torvalds layers = idp->layers; 2161da177e4SLinus Torvalds if (unlikely(!p)) { 2174ae53789SNadia Derbey if (!(p = get_from_free_list(idp))) 2181da177e4SLinus Torvalds return -1; 2196ff2d39bSManfred Spraul p->layer = 0; 2201da177e4SLinus Torvalds layers = 1; 2211da177e4SLinus Torvalds } 2221da177e4SLinus Torvalds /* 2231da177e4SLinus Torvalds * Add a new layer to the top of the tree if the requested 2241da177e4SLinus Torvalds * id is larger than the currently allocated space. 2251da177e4SLinus Torvalds */ 226125c4c70SFengguang Wu while ((layers < (MAX_IDR_LEVEL - 1)) && (id >= (1 << (layers*IDR_BITS)))) { 2271da177e4SLinus Torvalds layers++; 228711a49a0SManfred Spraul if (!p->count) { 229711a49a0SManfred Spraul /* special case: if the tree is currently empty, 230711a49a0SManfred Spraul * then we grow the tree by moving the top node 231711a49a0SManfred Spraul * upwards. 232711a49a0SManfred Spraul */ 233711a49a0SManfred Spraul p->layer++; 2341da177e4SLinus Torvalds continue; 235711a49a0SManfred Spraul } 2364ae53789SNadia Derbey if (!(new = get_from_free_list(idp))) { 2371da177e4SLinus Torvalds /* 2381da177e4SLinus Torvalds * The allocation failed. If we built part of 2391da177e4SLinus Torvalds * the structure tear it down. 2401da177e4SLinus Torvalds */ 241c259cc28SRoland Dreier spin_lock_irqsave(&idp->lock, flags); 2421da177e4SLinus Torvalds for (new = p; p && p != idp->top; new = p) { 2431da177e4SLinus Torvalds p = p->ary[0]; 2441da177e4SLinus Torvalds new->ary[0] = NULL; 2451da177e4SLinus Torvalds new->bitmap = new->count = 0; 2464ae53789SNadia Derbey __move_to_free_list(idp, new); 2471da177e4SLinus Torvalds } 248c259cc28SRoland Dreier spin_unlock_irqrestore(&idp->lock, flags); 2491da177e4SLinus Torvalds return -1; 2501da177e4SLinus Torvalds } 2511da177e4SLinus Torvalds new->ary[0] = p; 2521da177e4SLinus Torvalds new->count = 1; 2536ff2d39bSManfred Spraul new->layer = layers-1; 2541da177e4SLinus Torvalds if (p->bitmap == IDR_FULL) 2551da177e4SLinus Torvalds __set_bit(0, &new->bitmap); 2561da177e4SLinus Torvalds p = new; 2571da177e4SLinus Torvalds } 2583219b3b7SNadia Derbey rcu_assign_pointer(idp->top, p); 2591da177e4SLinus Torvalds idp->layers = layers; 260e33ac8bdSTejun Heo v = sub_alloc(idp, &id, pa); 261944ca05cSNadia Derbey if (v == IDR_NEED_TO_GROW) 2621da177e4SLinus Torvalds goto build_up; 2631da177e4SLinus Torvalds return(v); 2641da177e4SLinus Torvalds } 2651da177e4SLinus Torvalds 266e33ac8bdSTejun Heo static int idr_get_new_above_int(struct idr *idp, void *ptr, int starting_id) 267e33ac8bdSTejun Heo { 268125c4c70SFengguang Wu struct idr_layer *pa[MAX_IDR_LEVEL]; 269e33ac8bdSTejun Heo int id; 270e33ac8bdSTejun Heo 271e33ac8bdSTejun Heo id = idr_get_empty_slot(idp, starting_id, pa); 272e33ac8bdSTejun Heo if (id >= 0) { 273e33ac8bdSTejun Heo /* 274e33ac8bdSTejun Heo * Successfully found an empty slot. Install the user 275e33ac8bdSTejun Heo * pointer and mark the slot full. 276e33ac8bdSTejun Heo */ 2773219b3b7SNadia Derbey rcu_assign_pointer(pa[0]->ary[id & IDR_MASK], 2783219b3b7SNadia Derbey (struct idr_layer *)ptr); 279e33ac8bdSTejun Heo pa[0]->count++; 280e33ac8bdSTejun Heo idr_mark_full(pa, id); 281e33ac8bdSTejun Heo } 282e33ac8bdSTejun Heo 283e33ac8bdSTejun Heo return id; 284e33ac8bdSTejun Heo } 285e33ac8bdSTejun Heo 2861da177e4SLinus Torvalds /** 2877c657f2fSJohn McCutchan * idr_get_new_above - allocate new idr entry above or equal to a start id 2881da177e4SLinus Torvalds * @idp: idr handle 28994e2bd68SThadeu Lima de Souza Cascardo * @ptr: pointer you want associated with the id 290ea24ea85SNaohiro Aota * @starting_id: id to start search at 2911da177e4SLinus Torvalds * @id: pointer to the allocated handle 2921da177e4SLinus Torvalds * 2931da177e4SLinus Torvalds * This is the allocate id function. It should be called with any 2941da177e4SLinus Torvalds * required locks. 2951da177e4SLinus Torvalds * 296066a9be6SNaohiro Aota * If allocation from IDR's private freelist fails, idr_get_new_above() will 29756083ab1SRandy Dunlap * return %-EAGAIN. The caller should retry the idr_pre_get() call to refill 298066a9be6SNaohiro Aota * IDR's preallocation and then retry the idr_get_new_above() call. 299066a9be6SNaohiro Aota * 30056083ab1SRandy Dunlap * If the idr is full idr_get_new_above() will return %-ENOSPC. 3011da177e4SLinus Torvalds * 30256083ab1SRandy Dunlap * @id returns a value in the range @starting_id ... %0x7fffffff 3031da177e4SLinus Torvalds */ 3041da177e4SLinus Torvalds int idr_get_new_above(struct idr *idp, void *ptr, int starting_id, int *id) 3051da177e4SLinus Torvalds { 3061da177e4SLinus Torvalds int rv; 307e15ae2ddSJesper Juhl 3081da177e4SLinus Torvalds rv = idr_get_new_above_int(idp, ptr, starting_id); 3091da177e4SLinus Torvalds /* 3101da177e4SLinus Torvalds * This is a cheap hack until the IDR code can be fixed to 3111da177e4SLinus Torvalds * return proper error values. 3121da177e4SLinus Torvalds */ 313944ca05cSNadia Derbey if (rv < 0) 314944ca05cSNadia Derbey return _idr_rc_to_errno(rv); 3151da177e4SLinus Torvalds *id = rv; 3161da177e4SLinus Torvalds return 0; 3171da177e4SLinus Torvalds } 3181da177e4SLinus Torvalds EXPORT_SYMBOL(idr_get_new_above); 3191da177e4SLinus Torvalds 3201da177e4SLinus Torvalds /** 3211da177e4SLinus Torvalds * idr_get_new - allocate new idr entry 3221da177e4SLinus Torvalds * @idp: idr handle 32394e2bd68SThadeu Lima de Souza Cascardo * @ptr: pointer you want associated with the id 3241da177e4SLinus Torvalds * @id: pointer to the allocated handle 3251da177e4SLinus Torvalds * 326066a9be6SNaohiro Aota * If allocation from IDR's private freelist fails, idr_get_new_above() will 32756083ab1SRandy Dunlap * return %-EAGAIN. The caller should retry the idr_pre_get() call to refill 328066a9be6SNaohiro Aota * IDR's preallocation and then retry the idr_get_new_above() call. 3291da177e4SLinus Torvalds * 33056083ab1SRandy Dunlap * If the idr is full idr_get_new_above() will return %-ENOSPC. 3311da177e4SLinus Torvalds * 33256083ab1SRandy Dunlap * @id returns a value in the range %0 ... %0x7fffffff 3331da177e4SLinus Torvalds */ 3341da177e4SLinus Torvalds int idr_get_new(struct idr *idp, void *ptr, int *id) 3351da177e4SLinus Torvalds { 3361da177e4SLinus Torvalds int rv; 337e15ae2ddSJesper Juhl 3381da177e4SLinus Torvalds rv = idr_get_new_above_int(idp, ptr, 0); 3391da177e4SLinus Torvalds /* 3401da177e4SLinus Torvalds * This is a cheap hack until the IDR code can be fixed to 3411da177e4SLinus Torvalds * return proper error values. 3421da177e4SLinus Torvalds */ 343944ca05cSNadia Derbey if (rv < 0) 344944ca05cSNadia Derbey return _idr_rc_to_errno(rv); 3451da177e4SLinus Torvalds *id = rv; 3461da177e4SLinus Torvalds return 0; 3471da177e4SLinus Torvalds } 3481da177e4SLinus Torvalds EXPORT_SYMBOL(idr_get_new); 3491da177e4SLinus Torvalds 3501da177e4SLinus Torvalds static void idr_remove_warning(int id) 3511da177e4SLinus Torvalds { 352f098ad65SNadia Derbey printk(KERN_WARNING 353f098ad65SNadia Derbey "idr_remove called for id=%d which is not allocated.\n", id); 3541da177e4SLinus Torvalds dump_stack(); 3551da177e4SLinus Torvalds } 3561da177e4SLinus Torvalds 3571da177e4SLinus Torvalds static void sub_remove(struct idr *idp, int shift, int id) 3581da177e4SLinus Torvalds { 3591da177e4SLinus Torvalds struct idr_layer *p = idp->top; 360125c4c70SFengguang Wu struct idr_layer **pa[MAX_IDR_LEVEL]; 3611da177e4SLinus Torvalds struct idr_layer ***paa = &pa[0]; 362cf481c20SNadia Derbey struct idr_layer *to_free; 3631da177e4SLinus Torvalds int n; 3641da177e4SLinus Torvalds 3651da177e4SLinus Torvalds *paa = NULL; 3661da177e4SLinus Torvalds *++paa = &idp->top; 3671da177e4SLinus Torvalds 3681da177e4SLinus Torvalds while ((shift > 0) && p) { 3691da177e4SLinus Torvalds n = (id >> shift) & IDR_MASK; 3701da177e4SLinus Torvalds __clear_bit(n, &p->bitmap); 3711da177e4SLinus Torvalds *++paa = &p->ary[n]; 3721da177e4SLinus Torvalds p = p->ary[n]; 3731da177e4SLinus Torvalds shift -= IDR_BITS; 3741da177e4SLinus Torvalds } 3751da177e4SLinus Torvalds n = id & IDR_MASK; 3761da177e4SLinus Torvalds if (likely(p != NULL && test_bit(n, &p->bitmap))){ 3771da177e4SLinus Torvalds __clear_bit(n, &p->bitmap); 378cf481c20SNadia Derbey rcu_assign_pointer(p->ary[n], NULL); 379cf481c20SNadia Derbey to_free = NULL; 3801da177e4SLinus Torvalds while(*paa && ! --((**paa)->count)){ 381cf481c20SNadia Derbey if (to_free) 382cf481c20SNadia Derbey free_layer(to_free); 383cf481c20SNadia Derbey to_free = **paa; 3841da177e4SLinus Torvalds **paa-- = NULL; 3851da177e4SLinus Torvalds } 3861da177e4SLinus Torvalds if (!*paa) 3871da177e4SLinus Torvalds idp->layers = 0; 388cf481c20SNadia Derbey if (to_free) 389cf481c20SNadia Derbey free_layer(to_free); 390e15ae2ddSJesper Juhl } else 3911da177e4SLinus Torvalds idr_remove_warning(id); 3921da177e4SLinus Torvalds } 3931da177e4SLinus Torvalds 3941da177e4SLinus Torvalds /** 39556083ab1SRandy Dunlap * idr_remove - remove the given id and free its slot 39672fd4a35SRobert P. J. Day * @idp: idr handle 39772fd4a35SRobert P. J. Day * @id: unique key 3981da177e4SLinus Torvalds */ 3991da177e4SLinus Torvalds void idr_remove(struct idr *idp, int id) 4001da177e4SLinus Torvalds { 4011da177e4SLinus Torvalds struct idr_layer *p; 402cf481c20SNadia Derbey struct idr_layer *to_free; 4031da177e4SLinus Torvalds 4041da177e4SLinus Torvalds /* Mask off upper bits we don't use for the search. */ 405125c4c70SFengguang Wu id &= MAX_IDR_MASK; 4061da177e4SLinus Torvalds 4071da177e4SLinus Torvalds sub_remove(idp, (idp->layers - 1) * IDR_BITS, id); 408e15ae2ddSJesper Juhl if (idp->top && idp->top->count == 1 && (idp->layers > 1) && 409cf481c20SNadia Derbey idp->top->ary[0]) { 410cf481c20SNadia Derbey /* 411cf481c20SNadia Derbey * Single child at leftmost slot: we can shrink the tree. 412cf481c20SNadia Derbey * This level is not needed anymore since when layers are 413cf481c20SNadia Derbey * inserted, they are inserted at the top of the existing 414cf481c20SNadia Derbey * tree. 415cf481c20SNadia Derbey */ 416cf481c20SNadia Derbey to_free = idp->top; 4171da177e4SLinus Torvalds p = idp->top->ary[0]; 418cf481c20SNadia Derbey rcu_assign_pointer(idp->top, p); 4191da177e4SLinus Torvalds --idp->layers; 420cf481c20SNadia Derbey to_free->bitmap = to_free->count = 0; 421cf481c20SNadia Derbey free_layer(to_free); 4221da177e4SLinus Torvalds } 423125c4c70SFengguang Wu while (idp->id_free_cnt >= MAX_IDR_FREE) { 4244ae53789SNadia Derbey p = get_from_free_list(idp); 425cf481c20SNadia Derbey /* 426cf481c20SNadia Derbey * Note: we don't call the rcu callback here, since the only 427cf481c20SNadia Derbey * layers that fall into the freelist are those that have been 428cf481c20SNadia Derbey * preallocated. 429cf481c20SNadia Derbey */ 4301da177e4SLinus Torvalds kmem_cache_free(idr_layer_cache, p); 4311da177e4SLinus Torvalds } 432af8e2a4cSNadia Derbey return; 4331da177e4SLinus Torvalds } 4341da177e4SLinus Torvalds EXPORT_SYMBOL(idr_remove); 4351da177e4SLinus Torvalds 4361da177e4SLinus Torvalds /** 43723936cc0SKristian Hoegsberg * idr_remove_all - remove all ids from the given idr tree 43823936cc0SKristian Hoegsberg * @idp: idr handle 43923936cc0SKristian Hoegsberg */ 44023936cc0SKristian Hoegsberg void idr_remove_all(struct idr *idp) 44123936cc0SKristian Hoegsberg { 4426ace06dcSOleg Nesterov int n, id, max; 4432dcb22b3SImre Deak int bt_mask; 44423936cc0SKristian Hoegsberg struct idr_layer *p; 445125c4c70SFengguang Wu struct idr_layer *pa[MAX_IDR_LEVEL]; 44623936cc0SKristian Hoegsberg struct idr_layer **paa = &pa[0]; 44723936cc0SKristian Hoegsberg 44823936cc0SKristian Hoegsberg n = idp->layers * IDR_BITS; 44923936cc0SKristian Hoegsberg p = idp->top; 4501b23336aSPaul E. McKenney rcu_assign_pointer(idp->top, NULL); 45123936cc0SKristian Hoegsberg max = 1 << n; 45223936cc0SKristian Hoegsberg 45323936cc0SKristian Hoegsberg id = 0; 4546ace06dcSOleg Nesterov while (id < max) { 45523936cc0SKristian Hoegsberg while (n > IDR_BITS && p) { 45623936cc0SKristian Hoegsberg n -= IDR_BITS; 45723936cc0SKristian Hoegsberg *paa++ = p; 45823936cc0SKristian Hoegsberg p = p->ary[(id >> n) & IDR_MASK]; 45923936cc0SKristian Hoegsberg } 46023936cc0SKristian Hoegsberg 4612dcb22b3SImre Deak bt_mask = id; 46223936cc0SKristian Hoegsberg id += 1 << n; 4632dcb22b3SImre Deak /* Get the highest bit that the above add changed from 0->1. */ 4642dcb22b3SImre Deak while (n < fls(id ^ bt_mask)) { 465cf481c20SNadia Derbey if (p) 466cf481c20SNadia Derbey free_layer(p); 46723936cc0SKristian Hoegsberg n += IDR_BITS; 46823936cc0SKristian Hoegsberg p = *--paa; 46923936cc0SKristian Hoegsberg } 47023936cc0SKristian Hoegsberg } 47123936cc0SKristian Hoegsberg idp->layers = 0; 47223936cc0SKristian Hoegsberg } 47323936cc0SKristian Hoegsberg EXPORT_SYMBOL(idr_remove_all); 47423936cc0SKristian Hoegsberg 47523936cc0SKristian Hoegsberg /** 4768d3b3591SAndrew Morton * idr_destroy - release all cached layers within an idr tree 477ea24ea85SNaohiro Aota * @idp: idr handle 4789bb26bc1STejun Heo * 4799bb26bc1STejun Heo * Free all id mappings and all idp_layers. After this function, @idp is 4809bb26bc1STejun Heo * completely unused and can be freed / recycled. The caller is 4819bb26bc1STejun Heo * responsible for ensuring that no one else accesses @idp during or after 4829bb26bc1STejun Heo * idr_destroy(). 4839bb26bc1STejun Heo * 4849bb26bc1STejun Heo * A typical clean-up sequence for objects stored in an idr tree will use 4859bb26bc1STejun Heo * idr_for_each() to free all objects, if necessay, then idr_destroy() to 4869bb26bc1STejun Heo * free up the id mappings and cached idr_layers. 4878d3b3591SAndrew Morton */ 4888d3b3591SAndrew Morton void idr_destroy(struct idr *idp) 4898d3b3591SAndrew Morton { 4909bb26bc1STejun Heo idr_remove_all(idp); 4919bb26bc1STejun Heo 4928d3b3591SAndrew Morton while (idp->id_free_cnt) { 4934ae53789SNadia Derbey struct idr_layer *p = get_from_free_list(idp); 4948d3b3591SAndrew Morton kmem_cache_free(idr_layer_cache, p); 4958d3b3591SAndrew Morton } 4968d3b3591SAndrew Morton } 4978d3b3591SAndrew Morton EXPORT_SYMBOL(idr_destroy); 4988d3b3591SAndrew Morton 4998d3b3591SAndrew Morton /** 5001da177e4SLinus Torvalds * idr_find - return pointer for given id 5011da177e4SLinus Torvalds * @idp: idr handle 5021da177e4SLinus Torvalds * @id: lookup key 5031da177e4SLinus Torvalds * 5041da177e4SLinus Torvalds * Return the pointer given the id it has been registered with. A %NULL 5051da177e4SLinus Torvalds * return indicates that @id is not valid or you passed %NULL in 5061da177e4SLinus Torvalds * idr_get_new(). 5071da177e4SLinus Torvalds * 508f9c46d6eSNadia Derbey * This function can be called under rcu_read_lock(), given that the leaf 509f9c46d6eSNadia Derbey * pointers lifetimes are correctly managed. 5101da177e4SLinus Torvalds */ 5111da177e4SLinus Torvalds void *idr_find(struct idr *idp, int id) 5121da177e4SLinus Torvalds { 5131da177e4SLinus Torvalds int n; 5141da177e4SLinus Torvalds struct idr_layer *p; 5151da177e4SLinus Torvalds 51696be753aSPaul E. McKenney p = rcu_dereference_raw(idp->top); 5176ff2d39bSManfred Spraul if (!p) 5186ff2d39bSManfred Spraul return NULL; 5196ff2d39bSManfred Spraul n = (p->layer+1) * IDR_BITS; 5201da177e4SLinus Torvalds 5211da177e4SLinus Torvalds /* Mask off upper bits we don't use for the search. */ 522125c4c70SFengguang Wu id &= MAX_IDR_MASK; 5231da177e4SLinus Torvalds 5241da177e4SLinus Torvalds if (id >= (1 << n)) 5251da177e4SLinus Torvalds return NULL; 5266ff2d39bSManfred Spraul BUG_ON(n == 0); 5271da177e4SLinus Torvalds 5281da177e4SLinus Torvalds while (n > 0 && p) { 5291da177e4SLinus Torvalds n -= IDR_BITS; 5306ff2d39bSManfred Spraul BUG_ON(n != p->layer*IDR_BITS); 53196be753aSPaul E. McKenney p = rcu_dereference_raw(p->ary[(id >> n) & IDR_MASK]); 5321da177e4SLinus Torvalds } 5331da177e4SLinus Torvalds return((void *)p); 5341da177e4SLinus Torvalds } 5351da177e4SLinus Torvalds EXPORT_SYMBOL(idr_find); 5361da177e4SLinus Torvalds 5375806f07cSJeff Mahoney /** 53896d7fa42SKristian Hoegsberg * idr_for_each - iterate through all stored pointers 53996d7fa42SKristian Hoegsberg * @idp: idr handle 54096d7fa42SKristian Hoegsberg * @fn: function to be called for each pointer 54196d7fa42SKristian Hoegsberg * @data: data passed back to callback function 54296d7fa42SKristian Hoegsberg * 54396d7fa42SKristian Hoegsberg * Iterate over the pointers registered with the given idr. The 54496d7fa42SKristian Hoegsberg * callback function will be called for each pointer currently 54596d7fa42SKristian Hoegsberg * registered, passing the id, the pointer and the data pointer passed 54696d7fa42SKristian Hoegsberg * to this function. It is not safe to modify the idr tree while in 54796d7fa42SKristian Hoegsberg * the callback, so functions such as idr_get_new and idr_remove are 54896d7fa42SKristian Hoegsberg * not allowed. 54996d7fa42SKristian Hoegsberg * 55096d7fa42SKristian Hoegsberg * We check the return of @fn each time. If it returns anything other 55156083ab1SRandy Dunlap * than %0, we break out and return that value. 55296d7fa42SKristian Hoegsberg * 55396d7fa42SKristian Hoegsberg * The caller must serialize idr_for_each() vs idr_get_new() and idr_remove(). 55496d7fa42SKristian Hoegsberg */ 55596d7fa42SKristian Hoegsberg int idr_for_each(struct idr *idp, 55696d7fa42SKristian Hoegsberg int (*fn)(int id, void *p, void *data), void *data) 55796d7fa42SKristian Hoegsberg { 55896d7fa42SKristian Hoegsberg int n, id, max, error = 0; 55996d7fa42SKristian Hoegsberg struct idr_layer *p; 560125c4c70SFengguang Wu struct idr_layer *pa[MAX_IDR_LEVEL]; 56196d7fa42SKristian Hoegsberg struct idr_layer **paa = &pa[0]; 56296d7fa42SKristian Hoegsberg 56396d7fa42SKristian Hoegsberg n = idp->layers * IDR_BITS; 56496be753aSPaul E. McKenney p = rcu_dereference_raw(idp->top); 56596d7fa42SKristian Hoegsberg max = 1 << n; 56696d7fa42SKristian Hoegsberg 56796d7fa42SKristian Hoegsberg id = 0; 56896d7fa42SKristian Hoegsberg while (id < max) { 56996d7fa42SKristian Hoegsberg while (n > 0 && p) { 57096d7fa42SKristian Hoegsberg n -= IDR_BITS; 57196d7fa42SKristian Hoegsberg *paa++ = p; 57296be753aSPaul E. McKenney p = rcu_dereference_raw(p->ary[(id >> n) & IDR_MASK]); 57396d7fa42SKristian Hoegsberg } 57496d7fa42SKristian Hoegsberg 57596d7fa42SKristian Hoegsberg if (p) { 57696d7fa42SKristian Hoegsberg error = fn(id, (void *)p, data); 57796d7fa42SKristian Hoegsberg if (error) 57896d7fa42SKristian Hoegsberg break; 57996d7fa42SKristian Hoegsberg } 58096d7fa42SKristian Hoegsberg 58196d7fa42SKristian Hoegsberg id += 1 << n; 58296d7fa42SKristian Hoegsberg while (n < fls(id)) { 58396d7fa42SKristian Hoegsberg n += IDR_BITS; 58496d7fa42SKristian Hoegsberg p = *--paa; 58596d7fa42SKristian Hoegsberg } 58696d7fa42SKristian Hoegsberg } 58796d7fa42SKristian Hoegsberg 58896d7fa42SKristian Hoegsberg return error; 58996d7fa42SKristian Hoegsberg } 59096d7fa42SKristian Hoegsberg EXPORT_SYMBOL(idr_for_each); 59196d7fa42SKristian Hoegsberg 59296d7fa42SKristian Hoegsberg /** 59338460b48SKAMEZAWA Hiroyuki * idr_get_next - lookup next object of id to given id. 59438460b48SKAMEZAWA Hiroyuki * @idp: idr handle 595ea24ea85SNaohiro Aota * @nextidp: pointer to lookup key 59638460b48SKAMEZAWA Hiroyuki * 59738460b48SKAMEZAWA Hiroyuki * Returns pointer to registered object with id, which is next number to 5981458ce16SNaohiro Aota * given id. After being looked up, *@nextidp will be updated for the next 5991458ce16SNaohiro Aota * iteration. 6009f7de827SHugh Dickins * 6019f7de827SHugh Dickins * This function can be called under rcu_read_lock(), given that the leaf 6029f7de827SHugh Dickins * pointers lifetimes are correctly managed. 60338460b48SKAMEZAWA Hiroyuki */ 60438460b48SKAMEZAWA Hiroyuki void *idr_get_next(struct idr *idp, int *nextidp) 60538460b48SKAMEZAWA Hiroyuki { 606125c4c70SFengguang Wu struct idr_layer *p, *pa[MAX_IDR_LEVEL]; 60738460b48SKAMEZAWA Hiroyuki struct idr_layer **paa = &pa[0]; 60838460b48SKAMEZAWA Hiroyuki int id = *nextidp; 60938460b48SKAMEZAWA Hiroyuki int n, max; 61038460b48SKAMEZAWA Hiroyuki 61138460b48SKAMEZAWA Hiroyuki /* find first ent */ 61294bfa3b6SPaul E. McKenney p = rcu_dereference_raw(idp->top); 61338460b48SKAMEZAWA Hiroyuki if (!p) 61438460b48SKAMEZAWA Hiroyuki return NULL; 6159f7de827SHugh Dickins n = (p->layer + 1) * IDR_BITS; 6169f7de827SHugh Dickins max = 1 << n; 61738460b48SKAMEZAWA Hiroyuki 61838460b48SKAMEZAWA Hiroyuki while (id < max) { 61938460b48SKAMEZAWA Hiroyuki while (n > 0 && p) { 62038460b48SKAMEZAWA Hiroyuki n -= IDR_BITS; 62138460b48SKAMEZAWA Hiroyuki *paa++ = p; 62294bfa3b6SPaul E. McKenney p = rcu_dereference_raw(p->ary[(id >> n) & IDR_MASK]); 62338460b48SKAMEZAWA Hiroyuki } 62438460b48SKAMEZAWA Hiroyuki 62538460b48SKAMEZAWA Hiroyuki if (p) { 62638460b48SKAMEZAWA Hiroyuki *nextidp = id; 62738460b48SKAMEZAWA Hiroyuki return p; 62838460b48SKAMEZAWA Hiroyuki } 62938460b48SKAMEZAWA Hiroyuki 6306cdae741STejun Heo /* 6316cdae741STejun Heo * Proceed to the next layer at the current level. Unlike 6326cdae741STejun Heo * idr_for_each(), @id isn't guaranteed to be aligned to 6336cdae741STejun Heo * layer boundary at this point and adding 1 << n may 6346cdae741STejun Heo * incorrectly skip IDs. Make sure we jump to the 6356cdae741STejun Heo * beginning of the next layer using round_up(). 6366cdae741STejun Heo */ 6376cdae741STejun Heo id = round_up(id + 1, 1 << n); 63838460b48SKAMEZAWA Hiroyuki while (n < fls(id)) { 63938460b48SKAMEZAWA Hiroyuki n += IDR_BITS; 64038460b48SKAMEZAWA Hiroyuki p = *--paa; 64138460b48SKAMEZAWA Hiroyuki } 64238460b48SKAMEZAWA Hiroyuki } 64338460b48SKAMEZAWA Hiroyuki return NULL; 64438460b48SKAMEZAWA Hiroyuki } 6454d1ee80fSBen Hutchings EXPORT_SYMBOL(idr_get_next); 64638460b48SKAMEZAWA Hiroyuki 64738460b48SKAMEZAWA Hiroyuki 64838460b48SKAMEZAWA Hiroyuki /** 6495806f07cSJeff Mahoney * idr_replace - replace pointer for given id 6505806f07cSJeff Mahoney * @idp: idr handle 6515806f07cSJeff Mahoney * @ptr: pointer you want associated with the id 6525806f07cSJeff Mahoney * @id: lookup key 6535806f07cSJeff Mahoney * 6545806f07cSJeff Mahoney * Replace the pointer registered with an id and return the old value. 65556083ab1SRandy Dunlap * A %-ENOENT return indicates that @id was not found. 65656083ab1SRandy Dunlap * A %-EINVAL return indicates that @id was not within valid constraints. 6575806f07cSJeff Mahoney * 658cf481c20SNadia Derbey * The caller must serialize with writers. 6595806f07cSJeff Mahoney */ 6605806f07cSJeff Mahoney void *idr_replace(struct idr *idp, void *ptr, int id) 6615806f07cSJeff Mahoney { 6625806f07cSJeff Mahoney int n; 6635806f07cSJeff Mahoney struct idr_layer *p, *old_p; 6645806f07cSJeff Mahoney 6655806f07cSJeff Mahoney p = idp->top; 6666ff2d39bSManfred Spraul if (!p) 6676ff2d39bSManfred Spraul return ERR_PTR(-EINVAL); 6686ff2d39bSManfred Spraul 6696ff2d39bSManfred Spraul n = (p->layer+1) * IDR_BITS; 6705806f07cSJeff Mahoney 671125c4c70SFengguang Wu id &= MAX_IDR_MASK; 6725806f07cSJeff Mahoney 6735806f07cSJeff Mahoney if (id >= (1 << n)) 6745806f07cSJeff Mahoney return ERR_PTR(-EINVAL); 6755806f07cSJeff Mahoney 6765806f07cSJeff Mahoney n -= IDR_BITS; 6775806f07cSJeff Mahoney while ((n > 0) && p) { 6785806f07cSJeff Mahoney p = p->ary[(id >> n) & IDR_MASK]; 6795806f07cSJeff Mahoney n -= IDR_BITS; 6805806f07cSJeff Mahoney } 6815806f07cSJeff Mahoney 6825806f07cSJeff Mahoney n = id & IDR_MASK; 6835806f07cSJeff Mahoney if (unlikely(p == NULL || !test_bit(n, &p->bitmap))) 6845806f07cSJeff Mahoney return ERR_PTR(-ENOENT); 6855806f07cSJeff Mahoney 6865806f07cSJeff Mahoney old_p = p->ary[n]; 687cf481c20SNadia Derbey rcu_assign_pointer(p->ary[n], ptr); 6885806f07cSJeff Mahoney 6895806f07cSJeff Mahoney return old_p; 6905806f07cSJeff Mahoney } 6915806f07cSJeff Mahoney EXPORT_SYMBOL(idr_replace); 6925806f07cSJeff Mahoney 693199f0ca5SAkinobu Mita void __init idr_init_cache(void) 6941da177e4SLinus Torvalds { 6951da177e4SLinus Torvalds idr_layer_cache = kmem_cache_create("idr_layer_cache", 6965b019e99SAndrew Morton sizeof(struct idr_layer), 0, SLAB_PANIC, NULL); 6971da177e4SLinus Torvalds } 6981da177e4SLinus Torvalds 6991da177e4SLinus Torvalds /** 7001da177e4SLinus Torvalds * idr_init - initialize idr handle 7011da177e4SLinus Torvalds * @idp: idr handle 7021da177e4SLinus Torvalds * 7031da177e4SLinus Torvalds * This function is use to set up the handle (@idp) that you will pass 7041da177e4SLinus Torvalds * to the rest of the functions. 7051da177e4SLinus Torvalds */ 7061da177e4SLinus Torvalds void idr_init(struct idr *idp) 7071da177e4SLinus Torvalds { 7081da177e4SLinus Torvalds memset(idp, 0, sizeof(struct idr)); 7091da177e4SLinus Torvalds spin_lock_init(&idp->lock); 7101da177e4SLinus Torvalds } 7111da177e4SLinus Torvalds EXPORT_SYMBOL(idr_init); 71272dba584STejun Heo 71372dba584STejun Heo 71456083ab1SRandy Dunlap /** 71556083ab1SRandy Dunlap * DOC: IDA description 71672dba584STejun Heo * IDA - IDR based ID allocator 71772dba584STejun Heo * 71856083ab1SRandy Dunlap * This is id allocator without id -> pointer translation. Memory 71972dba584STejun Heo * usage is much lower than full blown idr because each id only 72072dba584STejun Heo * occupies a bit. ida uses a custom leaf node which contains 72172dba584STejun Heo * IDA_BITMAP_BITS slots. 72272dba584STejun Heo * 72372dba584STejun Heo * 2007-04-25 written by Tejun Heo <htejun@gmail.com> 72472dba584STejun Heo */ 72572dba584STejun Heo 72672dba584STejun Heo static void free_bitmap(struct ida *ida, struct ida_bitmap *bitmap) 72772dba584STejun Heo { 72872dba584STejun Heo unsigned long flags; 72972dba584STejun Heo 73072dba584STejun Heo if (!ida->free_bitmap) { 73172dba584STejun Heo spin_lock_irqsave(&ida->idr.lock, flags); 73272dba584STejun Heo if (!ida->free_bitmap) { 73372dba584STejun Heo ida->free_bitmap = bitmap; 73472dba584STejun Heo bitmap = NULL; 73572dba584STejun Heo } 73672dba584STejun Heo spin_unlock_irqrestore(&ida->idr.lock, flags); 73772dba584STejun Heo } 73872dba584STejun Heo 73972dba584STejun Heo kfree(bitmap); 74072dba584STejun Heo } 74172dba584STejun Heo 74272dba584STejun Heo /** 74372dba584STejun Heo * ida_pre_get - reserve resources for ida allocation 74472dba584STejun Heo * @ida: ida handle 74572dba584STejun Heo * @gfp_mask: memory allocation flag 74672dba584STejun Heo * 74772dba584STejun Heo * This function should be called prior to locking and calling the 74872dba584STejun Heo * following function. It preallocates enough memory to satisfy the 74972dba584STejun Heo * worst possible allocation. 75072dba584STejun Heo * 75156083ab1SRandy Dunlap * If the system is REALLY out of memory this function returns %0, 75256083ab1SRandy Dunlap * otherwise %1. 75372dba584STejun Heo */ 75472dba584STejun Heo int ida_pre_get(struct ida *ida, gfp_t gfp_mask) 75572dba584STejun Heo { 75672dba584STejun Heo /* allocate idr_layers */ 75772dba584STejun Heo if (!idr_pre_get(&ida->idr, gfp_mask)) 75872dba584STejun Heo return 0; 75972dba584STejun Heo 76072dba584STejun Heo /* allocate free_bitmap */ 76172dba584STejun Heo if (!ida->free_bitmap) { 76272dba584STejun Heo struct ida_bitmap *bitmap; 76372dba584STejun Heo 76472dba584STejun Heo bitmap = kmalloc(sizeof(struct ida_bitmap), gfp_mask); 76572dba584STejun Heo if (!bitmap) 76672dba584STejun Heo return 0; 76772dba584STejun Heo 76872dba584STejun Heo free_bitmap(ida, bitmap); 76972dba584STejun Heo } 77072dba584STejun Heo 77172dba584STejun Heo return 1; 77272dba584STejun Heo } 77372dba584STejun Heo EXPORT_SYMBOL(ida_pre_get); 77472dba584STejun Heo 77572dba584STejun Heo /** 77672dba584STejun Heo * ida_get_new_above - allocate new ID above or equal to a start id 77772dba584STejun Heo * @ida: ida handle 778ea24ea85SNaohiro Aota * @starting_id: id to start search at 77972dba584STejun Heo * @p_id: pointer to the allocated handle 78072dba584STejun Heo * 781e3816c54SWang Sheng-Hui * Allocate new ID above or equal to @starting_id. It should be called 782e3816c54SWang Sheng-Hui * with any required locks. 78372dba584STejun Heo * 78456083ab1SRandy Dunlap * If memory is required, it will return %-EAGAIN, you should unlock 78572dba584STejun Heo * and go back to the ida_pre_get() call. If the ida is full, it will 78656083ab1SRandy Dunlap * return %-ENOSPC. 78772dba584STejun Heo * 78856083ab1SRandy Dunlap * @p_id returns a value in the range @starting_id ... %0x7fffffff. 78972dba584STejun Heo */ 79072dba584STejun Heo int ida_get_new_above(struct ida *ida, int starting_id, int *p_id) 79172dba584STejun Heo { 792125c4c70SFengguang Wu struct idr_layer *pa[MAX_IDR_LEVEL]; 79372dba584STejun Heo struct ida_bitmap *bitmap; 79472dba584STejun Heo unsigned long flags; 79572dba584STejun Heo int idr_id = starting_id / IDA_BITMAP_BITS; 79672dba584STejun Heo int offset = starting_id % IDA_BITMAP_BITS; 79772dba584STejun Heo int t, id; 79872dba584STejun Heo 79972dba584STejun Heo restart: 80072dba584STejun Heo /* get vacant slot */ 80172dba584STejun Heo t = idr_get_empty_slot(&ida->idr, idr_id, pa); 802944ca05cSNadia Derbey if (t < 0) 803944ca05cSNadia Derbey return _idr_rc_to_errno(t); 80472dba584STejun Heo 805125c4c70SFengguang Wu if (t * IDA_BITMAP_BITS >= MAX_IDR_BIT) 80672dba584STejun Heo return -ENOSPC; 80772dba584STejun Heo 80872dba584STejun Heo if (t != idr_id) 80972dba584STejun Heo offset = 0; 81072dba584STejun Heo idr_id = t; 81172dba584STejun Heo 81272dba584STejun Heo /* if bitmap isn't there, create a new one */ 81372dba584STejun Heo bitmap = (void *)pa[0]->ary[idr_id & IDR_MASK]; 81472dba584STejun Heo if (!bitmap) { 81572dba584STejun Heo spin_lock_irqsave(&ida->idr.lock, flags); 81672dba584STejun Heo bitmap = ida->free_bitmap; 81772dba584STejun Heo ida->free_bitmap = NULL; 81872dba584STejun Heo spin_unlock_irqrestore(&ida->idr.lock, flags); 81972dba584STejun Heo 82072dba584STejun Heo if (!bitmap) 82172dba584STejun Heo return -EAGAIN; 82272dba584STejun Heo 82372dba584STejun Heo memset(bitmap, 0, sizeof(struct ida_bitmap)); 8243219b3b7SNadia Derbey rcu_assign_pointer(pa[0]->ary[idr_id & IDR_MASK], 8253219b3b7SNadia Derbey (void *)bitmap); 82672dba584STejun Heo pa[0]->count++; 82772dba584STejun Heo } 82872dba584STejun Heo 82972dba584STejun Heo /* lookup for empty slot */ 83072dba584STejun Heo t = find_next_zero_bit(bitmap->bitmap, IDA_BITMAP_BITS, offset); 83172dba584STejun Heo if (t == IDA_BITMAP_BITS) { 83272dba584STejun Heo /* no empty slot after offset, continue to the next chunk */ 83372dba584STejun Heo idr_id++; 83472dba584STejun Heo offset = 0; 83572dba584STejun Heo goto restart; 83672dba584STejun Heo } 83772dba584STejun Heo 83872dba584STejun Heo id = idr_id * IDA_BITMAP_BITS + t; 839125c4c70SFengguang Wu if (id >= MAX_IDR_BIT) 84072dba584STejun Heo return -ENOSPC; 84172dba584STejun Heo 84272dba584STejun Heo __set_bit(t, bitmap->bitmap); 84372dba584STejun Heo if (++bitmap->nr_busy == IDA_BITMAP_BITS) 84472dba584STejun Heo idr_mark_full(pa, idr_id); 84572dba584STejun Heo 84672dba584STejun Heo *p_id = id; 84772dba584STejun Heo 84872dba584STejun Heo /* Each leaf node can handle nearly a thousand slots and the 84972dba584STejun Heo * whole idea of ida is to have small memory foot print. 85072dba584STejun Heo * Throw away extra resources one by one after each successful 85172dba584STejun Heo * allocation. 85272dba584STejun Heo */ 85372dba584STejun Heo if (ida->idr.id_free_cnt || ida->free_bitmap) { 8544ae53789SNadia Derbey struct idr_layer *p = get_from_free_list(&ida->idr); 85572dba584STejun Heo if (p) 85672dba584STejun Heo kmem_cache_free(idr_layer_cache, p); 85772dba584STejun Heo } 85872dba584STejun Heo 85972dba584STejun Heo return 0; 86072dba584STejun Heo } 86172dba584STejun Heo EXPORT_SYMBOL(ida_get_new_above); 86272dba584STejun Heo 86372dba584STejun Heo /** 86472dba584STejun Heo * ida_get_new - allocate new ID 86572dba584STejun Heo * @ida: idr handle 86672dba584STejun Heo * @p_id: pointer to the allocated handle 86772dba584STejun Heo * 86872dba584STejun Heo * Allocate new ID. It should be called with any required locks. 86972dba584STejun Heo * 87056083ab1SRandy Dunlap * If memory is required, it will return %-EAGAIN, you should unlock 87172dba584STejun Heo * and go back to the idr_pre_get() call. If the idr is full, it will 87256083ab1SRandy Dunlap * return %-ENOSPC. 87372dba584STejun Heo * 874f5c3dd71SPaul Bolle * @p_id returns a value in the range %0 ... %0x7fffffff. 87572dba584STejun Heo */ 87672dba584STejun Heo int ida_get_new(struct ida *ida, int *p_id) 87772dba584STejun Heo { 87872dba584STejun Heo return ida_get_new_above(ida, 0, p_id); 87972dba584STejun Heo } 88072dba584STejun Heo EXPORT_SYMBOL(ida_get_new); 88172dba584STejun Heo 88272dba584STejun Heo /** 88372dba584STejun Heo * ida_remove - remove the given ID 88472dba584STejun Heo * @ida: ida handle 88572dba584STejun Heo * @id: ID to free 88672dba584STejun Heo */ 88772dba584STejun Heo void ida_remove(struct ida *ida, int id) 88872dba584STejun Heo { 88972dba584STejun Heo struct idr_layer *p = ida->idr.top; 89072dba584STejun Heo int shift = (ida->idr.layers - 1) * IDR_BITS; 89172dba584STejun Heo int idr_id = id / IDA_BITMAP_BITS; 89272dba584STejun Heo int offset = id % IDA_BITMAP_BITS; 89372dba584STejun Heo int n; 89472dba584STejun Heo struct ida_bitmap *bitmap; 89572dba584STejun Heo 89672dba584STejun Heo /* clear full bits while looking up the leaf idr_layer */ 89772dba584STejun Heo while ((shift > 0) && p) { 89872dba584STejun Heo n = (idr_id >> shift) & IDR_MASK; 89972dba584STejun Heo __clear_bit(n, &p->bitmap); 90072dba584STejun Heo p = p->ary[n]; 90172dba584STejun Heo shift -= IDR_BITS; 90272dba584STejun Heo } 90372dba584STejun Heo 90472dba584STejun Heo if (p == NULL) 90572dba584STejun Heo goto err; 90672dba584STejun Heo 90772dba584STejun Heo n = idr_id & IDR_MASK; 90872dba584STejun Heo __clear_bit(n, &p->bitmap); 90972dba584STejun Heo 91072dba584STejun Heo bitmap = (void *)p->ary[n]; 91172dba584STejun Heo if (!test_bit(offset, bitmap->bitmap)) 91272dba584STejun Heo goto err; 91372dba584STejun Heo 91472dba584STejun Heo /* update bitmap and remove it if empty */ 91572dba584STejun Heo __clear_bit(offset, bitmap->bitmap); 91672dba584STejun Heo if (--bitmap->nr_busy == 0) { 91772dba584STejun Heo __set_bit(n, &p->bitmap); /* to please idr_remove() */ 91872dba584STejun Heo idr_remove(&ida->idr, idr_id); 91972dba584STejun Heo free_bitmap(ida, bitmap); 92072dba584STejun Heo } 92172dba584STejun Heo 92272dba584STejun Heo return; 92372dba584STejun Heo 92472dba584STejun Heo err: 92572dba584STejun Heo printk(KERN_WARNING 92672dba584STejun Heo "ida_remove called for id=%d which is not allocated.\n", id); 92772dba584STejun Heo } 92872dba584STejun Heo EXPORT_SYMBOL(ida_remove); 92972dba584STejun Heo 93072dba584STejun Heo /** 93172dba584STejun Heo * ida_destroy - release all cached layers within an ida tree 932ea24ea85SNaohiro Aota * @ida: ida handle 93372dba584STejun Heo */ 93472dba584STejun Heo void ida_destroy(struct ida *ida) 93572dba584STejun Heo { 93672dba584STejun Heo idr_destroy(&ida->idr); 93772dba584STejun Heo kfree(ida->free_bitmap); 93872dba584STejun Heo } 93972dba584STejun Heo EXPORT_SYMBOL(ida_destroy); 94072dba584STejun Heo 94172dba584STejun Heo /** 94288eca020SRusty Russell * ida_simple_get - get a new id. 94388eca020SRusty Russell * @ida: the (initialized) ida. 94488eca020SRusty Russell * @start: the minimum id (inclusive, < 0x8000000) 94588eca020SRusty Russell * @end: the maximum id (exclusive, < 0x8000000 or 0) 94688eca020SRusty Russell * @gfp_mask: memory allocation flags 94788eca020SRusty Russell * 94888eca020SRusty Russell * Allocates an id in the range start <= id < end, or returns -ENOSPC. 94988eca020SRusty Russell * On memory allocation failure, returns -ENOMEM. 95088eca020SRusty Russell * 95188eca020SRusty Russell * Use ida_simple_remove() to get rid of an id. 95288eca020SRusty Russell */ 95388eca020SRusty Russell int ida_simple_get(struct ida *ida, unsigned int start, unsigned int end, 95488eca020SRusty Russell gfp_t gfp_mask) 95588eca020SRusty Russell { 95688eca020SRusty Russell int ret, id; 95788eca020SRusty Russell unsigned int max; 95846cbc1d3STejun Heo unsigned long flags; 95988eca020SRusty Russell 96088eca020SRusty Russell BUG_ON((int)start < 0); 96188eca020SRusty Russell BUG_ON((int)end < 0); 96288eca020SRusty Russell 96388eca020SRusty Russell if (end == 0) 96488eca020SRusty Russell max = 0x80000000; 96588eca020SRusty Russell else { 96688eca020SRusty Russell BUG_ON(end < start); 96788eca020SRusty Russell max = end - 1; 96888eca020SRusty Russell } 96988eca020SRusty Russell 97088eca020SRusty Russell again: 97188eca020SRusty Russell if (!ida_pre_get(ida, gfp_mask)) 97288eca020SRusty Russell return -ENOMEM; 97388eca020SRusty Russell 97446cbc1d3STejun Heo spin_lock_irqsave(&simple_ida_lock, flags); 97588eca020SRusty Russell ret = ida_get_new_above(ida, start, &id); 97688eca020SRusty Russell if (!ret) { 97788eca020SRusty Russell if (id > max) { 97888eca020SRusty Russell ida_remove(ida, id); 97988eca020SRusty Russell ret = -ENOSPC; 98088eca020SRusty Russell } else { 98188eca020SRusty Russell ret = id; 98288eca020SRusty Russell } 98388eca020SRusty Russell } 98446cbc1d3STejun Heo spin_unlock_irqrestore(&simple_ida_lock, flags); 98588eca020SRusty Russell 98688eca020SRusty Russell if (unlikely(ret == -EAGAIN)) 98788eca020SRusty Russell goto again; 98888eca020SRusty Russell 98988eca020SRusty Russell return ret; 99088eca020SRusty Russell } 99188eca020SRusty Russell EXPORT_SYMBOL(ida_simple_get); 99288eca020SRusty Russell 99388eca020SRusty Russell /** 99488eca020SRusty Russell * ida_simple_remove - remove an allocated id. 99588eca020SRusty Russell * @ida: the (initialized) ida. 99688eca020SRusty Russell * @id: the id returned by ida_simple_get. 99788eca020SRusty Russell */ 99888eca020SRusty Russell void ida_simple_remove(struct ida *ida, unsigned int id) 99988eca020SRusty Russell { 100046cbc1d3STejun Heo unsigned long flags; 100146cbc1d3STejun Heo 100288eca020SRusty Russell BUG_ON((int)id < 0); 100346cbc1d3STejun Heo spin_lock_irqsave(&simple_ida_lock, flags); 100488eca020SRusty Russell ida_remove(ida, id); 100546cbc1d3STejun Heo spin_unlock_irqrestore(&simple_ida_lock, flags); 100688eca020SRusty Russell } 100788eca020SRusty Russell EXPORT_SYMBOL(ida_simple_remove); 100888eca020SRusty Russell 100988eca020SRusty Russell /** 101072dba584STejun Heo * ida_init - initialize ida handle 101172dba584STejun Heo * @ida: ida handle 101272dba584STejun Heo * 101372dba584STejun Heo * This function is use to set up the handle (@ida) that you will pass 101472dba584STejun Heo * to the rest of the functions. 101572dba584STejun Heo */ 101672dba584STejun Heo void ida_init(struct ida *ida) 101772dba584STejun Heo { 101872dba584STejun Heo memset(ida, 0, sizeof(struct ida)); 101972dba584STejun Heo idr_init(&ida->idr); 102072dba584STejun Heo 102172dba584STejun Heo } 102272dba584STejun Heo EXPORT_SYMBOL(ida_init); 1023