1 /* 2 * linux/mm/slab.c 3 * Written by Mark Hemment, 1996/97. 4 * (markhe@nextd.demon.co.uk) 5 * 6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli 7 * 8 * Major cleanup, different bufctl logic, per-cpu arrays 9 * (c) 2000 Manfred Spraul 10 * 11 * Cleanup, make the head arrays unconditional, preparation for NUMA 12 * (c) 2002 Manfred Spraul 13 * 14 * An implementation of the Slab Allocator as described in outline in; 15 * UNIX Internals: The New Frontiers by Uresh Vahalia 16 * Pub: Prentice Hall ISBN 0-13-101908-2 17 * or with a little more detail in; 18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator 19 * Jeff Bonwick (Sun Microsystems). 20 * Presented at: USENIX Summer 1994 Technical Conference 21 * 22 * The memory is organized in caches, one cache for each object type. 23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct) 24 * Each cache consists out of many slabs (they are small (usually one 25 * page long) and always contiguous), and each slab contains multiple 26 * initialized objects. 27 * 28 * This means, that your constructor is used only for newly allocated 29 * slabs and you must pass objects with the same initializations to 30 * kmem_cache_free. 31 * 32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM, 33 * normal). If you need a special memory type, then must create a new 34 * cache for that memory type. 35 * 36 * In order to reduce fragmentation, the slabs are sorted in 3 groups: 37 * full slabs with 0 free objects 38 * partial slabs 39 * empty slabs with no allocated objects 40 * 41 * If partial slabs exist, then new allocations come from these slabs, 42 * otherwise from empty slabs or new slabs are allocated. 43 * 44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache 45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs. 46 * 47 * Each cache has a short per-cpu head array, most allocs 48 * and frees go into that array, and if that array overflows, then 1/2 49 * of the entries in the array are given back into the global cache. 50 * The head array is strictly LIFO and should improve the cache hit rates. 51 * On SMP, it additionally reduces the spinlock operations. 52 * 53 * The c_cpuarray may not be read with enabled local interrupts - 54 * it's changed with a smp_call_function(). 55 * 56 * SMP synchronization: 57 * constructors and destructors are called without any locking. 58 * Several members in struct kmem_cache and struct slab never change, they 59 * are accessed without any locking. 60 * The per-cpu arrays are never accessed from the wrong cpu, no locking, 61 * and local interrupts are disabled so slab code is preempt-safe. 62 * The non-constant members are protected with a per-cache irq spinlock. 63 * 64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch 65 * in 2000 - many ideas in the current implementation are derived from 66 * his patch. 67 * 68 * Further notes from the original documentation: 69 * 70 * 11 April '97. Started multi-threading - markhe 71 * The global cache-chain is protected by the mutex 'cache_chain_mutex'. 72 * The sem is only needed when accessing/extending the cache-chain, which 73 * can never happen inside an interrupt (kmem_cache_create(), 74 * kmem_cache_shrink() and kmem_cache_reap()). 75 * 76 * At present, each engine can be growing a cache. This should be blocked. 77 * 78 * 15 March 2005. NUMA slab allocator. 79 * Shai Fultheim <shai@scalex86.org>. 80 * Shobhit Dayal <shobhit@calsoftinc.com> 81 * Alok N Kataria <alokk@calsoftinc.com> 82 * Christoph Lameter <christoph@lameter.com> 83 * 84 * Modified the slab allocator to be node aware on NUMA systems. 85 * Each node has its own list of partial, free and full slabs. 86 * All object allocations for a node occur from node specific slab lists. 87 */ 88 89 #include <linux/slab.h> 90 #include <linux/mm.h> 91 #include <linux/poison.h> 92 #include <linux/swap.h> 93 #include <linux/cache.h> 94 #include <linux/interrupt.h> 95 #include <linux/init.h> 96 #include <linux/compiler.h> 97 #include <linux/cpuset.h> 98 #include <linux/proc_fs.h> 99 #include <linux/seq_file.h> 100 #include <linux/notifier.h> 101 #include <linux/kallsyms.h> 102 #include <linux/cpu.h> 103 #include <linux/sysctl.h> 104 #include <linux/module.h> 105 #include <linux/rcupdate.h> 106 #include <linux/string.h> 107 #include <linux/uaccess.h> 108 #include <linux/nodemask.h> 109 #include <linux/kmemleak.h> 110 #include <linux/mempolicy.h> 111 #include <linux/mutex.h> 112 #include <linux/fault-inject.h> 113 #include <linux/rtmutex.h> 114 #include <linux/reciprocal_div.h> 115 #include <linux/debugobjects.h> 116 #include <linux/kmemcheck.h> 117 #include <linux/memory.h> 118 #include <linux/prefetch.h> 119 120 #include <asm/cacheflush.h> 121 #include <asm/tlbflush.h> 122 #include <asm/page.h> 123 124 #include <trace/events/kmem.h> 125 126 /* 127 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON. 128 * 0 for faster, smaller code (especially in the critical paths). 129 * 130 * STATS - 1 to collect stats for /proc/slabinfo. 131 * 0 for faster, smaller code (especially in the critical paths). 132 * 133 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible) 134 */ 135 136 #ifdef CONFIG_DEBUG_SLAB 137 #define DEBUG 1 138 #define STATS 1 139 #define FORCED_DEBUG 1 140 #else 141 #define DEBUG 0 142 #define STATS 0 143 #define FORCED_DEBUG 0 144 #endif 145 146 /* Shouldn't this be in a header file somewhere? */ 147 #define BYTES_PER_WORD sizeof(void *) 148 #define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long)) 149 150 #ifndef ARCH_KMALLOC_FLAGS 151 #define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN 152 #endif 153 154 /* Legal flag mask for kmem_cache_create(). */ 155 #if DEBUG 156 # define CREATE_MASK (SLAB_RED_ZONE | \ 157 SLAB_POISON | SLAB_HWCACHE_ALIGN | \ 158 SLAB_CACHE_DMA | \ 159 SLAB_STORE_USER | \ 160 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ 161 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \ 162 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK) 163 #else 164 # define CREATE_MASK (SLAB_HWCACHE_ALIGN | \ 165 SLAB_CACHE_DMA | \ 166 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ 167 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \ 168 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK) 169 #endif 170 171 /* 172 * kmem_bufctl_t: 173 * 174 * Bufctl's are used for linking objs within a slab 175 * linked offsets. 176 * 177 * This implementation relies on "struct page" for locating the cache & 178 * slab an object belongs to. 179 * This allows the bufctl structure to be small (one int), but limits 180 * the number of objects a slab (not a cache) can contain when off-slab 181 * bufctls are used. The limit is the size of the largest general cache 182 * that does not use off-slab slabs. 183 * For 32bit archs with 4 kB pages, is this 56. 184 * This is not serious, as it is only for large objects, when it is unwise 185 * to have too many per slab. 186 * Note: This limit can be raised by introducing a general cache whose size 187 * is less than 512 (PAGE_SIZE<<3), but greater than 256. 188 */ 189 190 typedef unsigned int kmem_bufctl_t; 191 #define BUFCTL_END (((kmem_bufctl_t)(~0U))-0) 192 #define BUFCTL_FREE (((kmem_bufctl_t)(~0U))-1) 193 #define BUFCTL_ACTIVE (((kmem_bufctl_t)(~0U))-2) 194 #define SLAB_LIMIT (((kmem_bufctl_t)(~0U))-3) 195 196 /* 197 * struct slab_rcu 198 * 199 * slab_destroy on a SLAB_DESTROY_BY_RCU cache uses this structure to 200 * arrange for kmem_freepages to be called via RCU. This is useful if 201 * we need to approach a kernel structure obliquely, from its address 202 * obtained without the usual locking. We can lock the structure to 203 * stabilize it and check it's still at the given address, only if we 204 * can be sure that the memory has not been meanwhile reused for some 205 * other kind of object (which our subsystem's lock might corrupt). 206 * 207 * rcu_read_lock before reading the address, then rcu_read_unlock after 208 * taking the spinlock within the structure expected at that address. 209 */ 210 struct slab_rcu { 211 struct rcu_head head; 212 struct kmem_cache *cachep; 213 void *addr; 214 }; 215 216 /* 217 * struct slab 218 * 219 * Manages the objs in a slab. Placed either at the beginning of mem allocated 220 * for a slab, or allocated from an general cache. 221 * Slabs are chained into three list: fully used, partial, fully free slabs. 222 */ 223 struct slab { 224 union { 225 struct { 226 struct list_head list; 227 unsigned long colouroff; 228 void *s_mem; /* including colour offset */ 229 unsigned int inuse; /* num of objs active in slab */ 230 kmem_bufctl_t free; 231 unsigned short nodeid; 232 }; 233 struct slab_rcu __slab_cover_slab_rcu; 234 }; 235 }; 236 237 /* 238 * struct array_cache 239 * 240 * Purpose: 241 * - LIFO ordering, to hand out cache-warm objects from _alloc 242 * - reduce the number of linked list operations 243 * - reduce spinlock operations 244 * 245 * The limit is stored in the per-cpu structure to reduce the data cache 246 * footprint. 247 * 248 */ 249 struct array_cache { 250 unsigned int avail; 251 unsigned int limit; 252 unsigned int batchcount; 253 unsigned int touched; 254 spinlock_t lock; 255 void *entry[]; /* 256 * Must have this definition in here for the proper 257 * alignment of array_cache. Also simplifies accessing 258 * the entries. 259 */ 260 }; 261 262 /* 263 * bootstrap: The caches do not work without cpuarrays anymore, but the 264 * cpuarrays are allocated from the generic caches... 265 */ 266 #define BOOT_CPUCACHE_ENTRIES 1 267 struct arraycache_init { 268 struct array_cache cache; 269 void *entries[BOOT_CPUCACHE_ENTRIES]; 270 }; 271 272 /* 273 * The slab lists for all objects. 274 */ 275 struct kmem_list3 { 276 struct list_head slabs_partial; /* partial list first, better asm code */ 277 struct list_head slabs_full; 278 struct list_head slabs_free; 279 unsigned long free_objects; 280 unsigned int free_limit; 281 unsigned int colour_next; /* Per-node cache coloring */ 282 spinlock_t list_lock; 283 struct array_cache *shared; /* shared per node */ 284 struct array_cache **alien; /* on other nodes */ 285 unsigned long next_reap; /* updated without locking */ 286 int free_touched; /* updated without locking */ 287 }; 288 289 /* 290 * Need this for bootstrapping a per node allocator. 291 */ 292 #define NUM_INIT_LISTS (3 * MAX_NUMNODES) 293 static struct kmem_list3 __initdata initkmem_list3[NUM_INIT_LISTS]; 294 #define CACHE_CACHE 0 295 #define SIZE_AC MAX_NUMNODES 296 #define SIZE_L3 (2 * MAX_NUMNODES) 297 298 static int drain_freelist(struct kmem_cache *cache, 299 struct kmem_list3 *l3, int tofree); 300 static void free_block(struct kmem_cache *cachep, void **objpp, int len, 301 int node); 302 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); 303 static void cache_reap(struct work_struct *unused); 304 305 /* 306 * This function must be completely optimized away if a constant is passed to 307 * it. Mostly the same as what is in linux/slab.h except it returns an index. 308 */ 309 static __always_inline int index_of(const size_t size) 310 { 311 extern void __bad_size(void); 312 313 if (__builtin_constant_p(size)) { 314 int i = 0; 315 316 #define CACHE(x) \ 317 if (size <=x) \ 318 return i; \ 319 else \ 320 i++; 321 #include <linux/kmalloc_sizes.h> 322 #undef CACHE 323 __bad_size(); 324 } else 325 __bad_size(); 326 return 0; 327 } 328 329 static int slab_early_init = 1; 330 331 #define INDEX_AC index_of(sizeof(struct arraycache_init)) 332 #define INDEX_L3 index_of(sizeof(struct kmem_list3)) 333 334 static void kmem_list3_init(struct kmem_list3 *parent) 335 { 336 INIT_LIST_HEAD(&parent->slabs_full); 337 INIT_LIST_HEAD(&parent->slabs_partial); 338 INIT_LIST_HEAD(&parent->slabs_free); 339 parent->shared = NULL; 340 parent->alien = NULL; 341 parent->colour_next = 0; 342 spin_lock_init(&parent->list_lock); 343 parent->free_objects = 0; 344 parent->free_touched = 0; 345 } 346 347 #define MAKE_LIST(cachep, listp, slab, nodeid) \ 348 do { \ 349 INIT_LIST_HEAD(listp); \ 350 list_splice(&(cachep->nodelists[nodeid]->slab), listp); \ 351 } while (0) 352 353 #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ 354 do { \ 355 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \ 356 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \ 357 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \ 358 } while (0) 359 360 #define CFLGS_OFF_SLAB (0x80000000UL) 361 #define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB) 362 363 #define BATCHREFILL_LIMIT 16 364 /* 365 * Optimization question: fewer reaps means less probability for unnessary 366 * cpucache drain/refill cycles. 367 * 368 * OTOH the cpuarrays can contain lots of objects, 369 * which could lock up otherwise freeable slabs. 370 */ 371 #define REAPTIMEOUT_CPUC (2*HZ) 372 #define REAPTIMEOUT_LIST3 (4*HZ) 373 374 #if STATS 375 #define STATS_INC_ACTIVE(x) ((x)->num_active++) 376 #define STATS_DEC_ACTIVE(x) ((x)->num_active--) 377 #define STATS_INC_ALLOCED(x) ((x)->num_allocations++) 378 #define STATS_INC_GROWN(x) ((x)->grown++) 379 #define STATS_ADD_REAPED(x,y) ((x)->reaped += (y)) 380 #define STATS_SET_HIGH(x) \ 381 do { \ 382 if ((x)->num_active > (x)->high_mark) \ 383 (x)->high_mark = (x)->num_active; \ 384 } while (0) 385 #define STATS_INC_ERR(x) ((x)->errors++) 386 #define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++) 387 #define STATS_INC_NODEFREES(x) ((x)->node_frees++) 388 #define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++) 389 #define STATS_SET_FREEABLE(x, i) \ 390 do { \ 391 if ((x)->max_freeable < i) \ 392 (x)->max_freeable = i; \ 393 } while (0) 394 #define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit) 395 #define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss) 396 #define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit) 397 #define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss) 398 #else 399 #define STATS_INC_ACTIVE(x) do { } while (0) 400 #define STATS_DEC_ACTIVE(x) do { } while (0) 401 #define STATS_INC_ALLOCED(x) do { } while (0) 402 #define STATS_INC_GROWN(x) do { } while (0) 403 #define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0) 404 #define STATS_SET_HIGH(x) do { } while (0) 405 #define STATS_INC_ERR(x) do { } while (0) 406 #define STATS_INC_NODEALLOCS(x) do { } while (0) 407 #define STATS_INC_NODEFREES(x) do { } while (0) 408 #define STATS_INC_ACOVERFLOW(x) do { } while (0) 409 #define STATS_SET_FREEABLE(x, i) do { } while (0) 410 #define STATS_INC_ALLOCHIT(x) do { } while (0) 411 #define STATS_INC_ALLOCMISS(x) do { } while (0) 412 #define STATS_INC_FREEHIT(x) do { } while (0) 413 #define STATS_INC_FREEMISS(x) do { } while (0) 414 #endif 415 416 #if DEBUG 417 418 /* 419 * memory layout of objects: 420 * 0 : objp 421 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that 422 * the end of an object is aligned with the end of the real 423 * allocation. Catches writes behind the end of the allocation. 424 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1: 425 * redzone word. 426 * cachep->obj_offset: The real object. 427 * cachep->buffer_size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] 428 * cachep->buffer_size - 1* BYTES_PER_WORD: last caller address 429 * [BYTES_PER_WORD long] 430 */ 431 static int obj_offset(struct kmem_cache *cachep) 432 { 433 return cachep->obj_offset; 434 } 435 436 static int obj_size(struct kmem_cache *cachep) 437 { 438 return cachep->obj_size; 439 } 440 441 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) 442 { 443 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 444 return (unsigned long long*) (objp + obj_offset(cachep) - 445 sizeof(unsigned long long)); 446 } 447 448 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) 449 { 450 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 451 if (cachep->flags & SLAB_STORE_USER) 452 return (unsigned long long *)(objp + cachep->buffer_size - 453 sizeof(unsigned long long) - 454 REDZONE_ALIGN); 455 return (unsigned long long *) (objp + cachep->buffer_size - 456 sizeof(unsigned long long)); 457 } 458 459 static void **dbg_userword(struct kmem_cache *cachep, void *objp) 460 { 461 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); 462 return (void **)(objp + cachep->buffer_size - BYTES_PER_WORD); 463 } 464 465 #else 466 467 #define obj_offset(x) 0 468 #define obj_size(cachep) (cachep->buffer_size) 469 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) 470 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) 471 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) 472 473 #endif 474 475 #ifdef CONFIG_TRACING 476 size_t slab_buffer_size(struct kmem_cache *cachep) 477 { 478 return cachep->buffer_size; 479 } 480 EXPORT_SYMBOL(slab_buffer_size); 481 #endif 482 483 /* 484 * Do not go above this order unless 0 objects fit into the slab or 485 * overridden on the command line. 486 */ 487 #define SLAB_MAX_ORDER_HI 1 488 #define SLAB_MAX_ORDER_LO 0 489 static int slab_max_order = SLAB_MAX_ORDER_LO; 490 static bool slab_max_order_set __initdata; 491 492 /* 493 * Functions for storing/retrieving the cachep and or slab from the page 494 * allocator. These are used to find the slab an obj belongs to. With kfree(), 495 * these are used to find the cache which an obj belongs to. 496 */ 497 static inline void page_set_cache(struct page *page, struct kmem_cache *cache) 498 { 499 page->lru.next = (struct list_head *)cache; 500 } 501 502 static inline struct kmem_cache *page_get_cache(struct page *page) 503 { 504 page = compound_head(page); 505 BUG_ON(!PageSlab(page)); 506 return (struct kmem_cache *)page->lru.next; 507 } 508 509 static inline void page_set_slab(struct page *page, struct slab *slab) 510 { 511 page->lru.prev = (struct list_head *)slab; 512 } 513 514 static inline struct slab *page_get_slab(struct page *page) 515 { 516 BUG_ON(!PageSlab(page)); 517 return (struct slab *)page->lru.prev; 518 } 519 520 static inline struct kmem_cache *virt_to_cache(const void *obj) 521 { 522 struct page *page = virt_to_head_page(obj); 523 return page_get_cache(page); 524 } 525 526 static inline struct slab *virt_to_slab(const void *obj) 527 { 528 struct page *page = virt_to_head_page(obj); 529 return page_get_slab(page); 530 } 531 532 static inline void *index_to_obj(struct kmem_cache *cache, struct slab *slab, 533 unsigned int idx) 534 { 535 return slab->s_mem + cache->buffer_size * idx; 536 } 537 538 /* 539 * We want to avoid an expensive divide : (offset / cache->buffer_size) 540 * Using the fact that buffer_size is a constant for a particular cache, 541 * we can replace (offset / cache->buffer_size) by 542 * reciprocal_divide(offset, cache->reciprocal_buffer_size) 543 */ 544 static inline unsigned int obj_to_index(const struct kmem_cache *cache, 545 const struct slab *slab, void *obj) 546 { 547 u32 offset = (obj - slab->s_mem); 548 return reciprocal_divide(offset, cache->reciprocal_buffer_size); 549 } 550 551 /* 552 * These are the default caches for kmalloc. Custom caches can have other sizes. 553 */ 554 struct cache_sizes malloc_sizes[] = { 555 #define CACHE(x) { .cs_size = (x) }, 556 #include <linux/kmalloc_sizes.h> 557 CACHE(ULONG_MAX) 558 #undef CACHE 559 }; 560 EXPORT_SYMBOL(malloc_sizes); 561 562 /* Must match cache_sizes above. Out of line to keep cache footprint low. */ 563 struct cache_names { 564 char *name; 565 char *name_dma; 566 }; 567 568 static struct cache_names __initdata cache_names[] = { 569 #define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, 570 #include <linux/kmalloc_sizes.h> 571 {NULL,} 572 #undef CACHE 573 }; 574 575 static struct arraycache_init initarray_cache __initdata = 576 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 577 static struct arraycache_init initarray_generic = 578 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 579 580 /* internal cache of cache description objs */ 581 static struct kmem_list3 *cache_cache_nodelists[MAX_NUMNODES]; 582 static struct kmem_cache cache_cache = { 583 .nodelists = cache_cache_nodelists, 584 .batchcount = 1, 585 .limit = BOOT_CPUCACHE_ENTRIES, 586 .shared = 1, 587 .buffer_size = sizeof(struct kmem_cache), 588 .name = "kmem_cache", 589 }; 590 591 #define BAD_ALIEN_MAGIC 0x01020304ul 592 593 /* 594 * chicken and egg problem: delay the per-cpu array allocation 595 * until the general caches are up. 596 */ 597 static enum { 598 NONE, 599 PARTIAL_AC, 600 PARTIAL_L3, 601 EARLY, 602 LATE, 603 FULL 604 } g_cpucache_up; 605 606 /* 607 * used by boot code to determine if it can use slab based allocator 608 */ 609 int slab_is_available(void) 610 { 611 return g_cpucache_up >= EARLY; 612 } 613 614 #ifdef CONFIG_LOCKDEP 615 616 /* 617 * Slab sometimes uses the kmalloc slabs to store the slab headers 618 * for other slabs "off slab". 619 * The locking for this is tricky in that it nests within the locks 620 * of all other slabs in a few places; to deal with this special 621 * locking we put on-slab caches into a separate lock-class. 622 * 623 * We set lock class for alien array caches which are up during init. 624 * The lock annotation will be lost if all cpus of a node goes down and 625 * then comes back up during hotplug 626 */ 627 static struct lock_class_key on_slab_l3_key; 628 static struct lock_class_key on_slab_alc_key; 629 630 static struct lock_class_key debugobj_l3_key; 631 static struct lock_class_key debugobj_alc_key; 632 633 static void slab_set_lock_classes(struct kmem_cache *cachep, 634 struct lock_class_key *l3_key, struct lock_class_key *alc_key, 635 int q) 636 { 637 struct array_cache **alc; 638 struct kmem_list3 *l3; 639 int r; 640 641 l3 = cachep->nodelists[q]; 642 if (!l3) 643 return; 644 645 lockdep_set_class(&l3->list_lock, l3_key); 646 alc = l3->alien; 647 /* 648 * FIXME: This check for BAD_ALIEN_MAGIC 649 * should go away when common slab code is taught to 650 * work even without alien caches. 651 * Currently, non NUMA code returns BAD_ALIEN_MAGIC 652 * for alloc_alien_cache, 653 */ 654 if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC) 655 return; 656 for_each_node(r) { 657 if (alc[r]) 658 lockdep_set_class(&alc[r]->lock, alc_key); 659 } 660 } 661 662 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) 663 { 664 slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node); 665 } 666 667 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) 668 { 669 int node; 670 671 for_each_online_node(node) 672 slab_set_debugobj_lock_classes_node(cachep, node); 673 } 674 675 static void init_node_lock_keys(int q) 676 { 677 struct cache_sizes *s = malloc_sizes; 678 679 if (g_cpucache_up < LATE) 680 return; 681 682 for (s = malloc_sizes; s->cs_size != ULONG_MAX; s++) { 683 struct kmem_list3 *l3; 684 685 l3 = s->cs_cachep->nodelists[q]; 686 if (!l3 || OFF_SLAB(s->cs_cachep)) 687 continue; 688 689 slab_set_lock_classes(s->cs_cachep, &on_slab_l3_key, 690 &on_slab_alc_key, q); 691 } 692 } 693 694 static inline void init_lock_keys(void) 695 { 696 int node; 697 698 for_each_node(node) 699 init_node_lock_keys(node); 700 } 701 #else 702 static void init_node_lock_keys(int q) 703 { 704 } 705 706 static inline void init_lock_keys(void) 707 { 708 } 709 710 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) 711 { 712 } 713 714 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) 715 { 716 } 717 #endif 718 719 /* 720 * Guard access to the cache-chain. 721 */ 722 static DEFINE_MUTEX(cache_chain_mutex); 723 static struct list_head cache_chain; 724 725 static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); 726 727 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) 728 { 729 return cachep->array[smp_processor_id()]; 730 } 731 732 static inline struct kmem_cache *__find_general_cachep(size_t size, 733 gfp_t gfpflags) 734 { 735 struct cache_sizes *csizep = malloc_sizes; 736 737 #if DEBUG 738 /* This happens if someone tries to call 739 * kmem_cache_create(), or __kmalloc(), before 740 * the generic caches are initialized. 741 */ 742 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); 743 #endif 744 if (!size) 745 return ZERO_SIZE_PTR; 746 747 while (size > csizep->cs_size) 748 csizep++; 749 750 /* 751 * Really subtle: The last entry with cs->cs_size==ULONG_MAX 752 * has cs_{dma,}cachep==NULL. Thus no special case 753 * for large kmalloc calls required. 754 */ 755 #ifdef CONFIG_ZONE_DMA 756 if (unlikely(gfpflags & GFP_DMA)) 757 return csizep->cs_dmacachep; 758 #endif 759 return csizep->cs_cachep; 760 } 761 762 static struct kmem_cache *kmem_find_general_cachep(size_t size, gfp_t gfpflags) 763 { 764 return __find_general_cachep(size, gfpflags); 765 } 766 767 static size_t slab_mgmt_size(size_t nr_objs, size_t align) 768 { 769 return ALIGN(sizeof(struct slab)+nr_objs*sizeof(kmem_bufctl_t), align); 770 } 771 772 /* 773 * Calculate the number of objects and left-over bytes for a given buffer size. 774 */ 775 static void cache_estimate(unsigned long gfporder, size_t buffer_size, 776 size_t align, int flags, size_t *left_over, 777 unsigned int *num) 778 { 779 int nr_objs; 780 size_t mgmt_size; 781 size_t slab_size = PAGE_SIZE << gfporder; 782 783 /* 784 * The slab management structure can be either off the slab or 785 * on it. For the latter case, the memory allocated for a 786 * slab is used for: 787 * 788 * - The struct slab 789 * - One kmem_bufctl_t for each object 790 * - Padding to respect alignment of @align 791 * - @buffer_size bytes for each object 792 * 793 * If the slab management structure is off the slab, then the 794 * alignment will already be calculated into the size. Because 795 * the slabs are all pages aligned, the objects will be at the 796 * correct alignment when allocated. 797 */ 798 if (flags & CFLGS_OFF_SLAB) { 799 mgmt_size = 0; 800 nr_objs = slab_size / buffer_size; 801 802 if (nr_objs > SLAB_LIMIT) 803 nr_objs = SLAB_LIMIT; 804 } else { 805 /* 806 * Ignore padding for the initial guess. The padding 807 * is at most @align-1 bytes, and @buffer_size is at 808 * least @align. In the worst case, this result will 809 * be one greater than the number of objects that fit 810 * into the memory allocation when taking the padding 811 * into account. 812 */ 813 nr_objs = (slab_size - sizeof(struct slab)) / 814 (buffer_size + sizeof(kmem_bufctl_t)); 815 816 /* 817 * This calculated number will be either the right 818 * amount, or one greater than what we want. 819 */ 820 if (slab_mgmt_size(nr_objs, align) + nr_objs*buffer_size 821 > slab_size) 822 nr_objs--; 823 824 if (nr_objs > SLAB_LIMIT) 825 nr_objs = SLAB_LIMIT; 826 827 mgmt_size = slab_mgmt_size(nr_objs, align); 828 } 829 *num = nr_objs; 830 *left_over = slab_size - nr_objs*buffer_size - mgmt_size; 831 } 832 833 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg) 834 835 static void __slab_error(const char *function, struct kmem_cache *cachep, 836 char *msg) 837 { 838 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", 839 function, cachep->name, msg); 840 dump_stack(); 841 } 842 843 /* 844 * By default on NUMA we use alien caches to stage the freeing of 845 * objects allocated from other nodes. This causes massive memory 846 * inefficiencies when using fake NUMA setup to split memory into a 847 * large number of small nodes, so it can be disabled on the command 848 * line 849 */ 850 851 static int use_alien_caches __read_mostly = 1; 852 static int __init noaliencache_setup(char *s) 853 { 854 use_alien_caches = 0; 855 return 1; 856 } 857 __setup("noaliencache", noaliencache_setup); 858 859 static int __init slab_max_order_setup(char *str) 860 { 861 get_option(&str, &slab_max_order); 862 slab_max_order = slab_max_order < 0 ? 0 : 863 min(slab_max_order, MAX_ORDER - 1); 864 slab_max_order_set = true; 865 866 return 1; 867 } 868 __setup("slab_max_order=", slab_max_order_setup); 869 870 #ifdef CONFIG_NUMA 871 /* 872 * Special reaping functions for NUMA systems called from cache_reap(). 873 * These take care of doing round robin flushing of alien caches (containing 874 * objects freed on different nodes from which they were allocated) and the 875 * flushing of remote pcps by calling drain_node_pages. 876 */ 877 static DEFINE_PER_CPU(unsigned long, slab_reap_node); 878 879 static void init_reap_node(int cpu) 880 { 881 int node; 882 883 node = next_node(cpu_to_mem(cpu), node_online_map); 884 if (node == MAX_NUMNODES) 885 node = first_node(node_online_map); 886 887 per_cpu(slab_reap_node, cpu) = node; 888 } 889 890 static void next_reap_node(void) 891 { 892 int node = __this_cpu_read(slab_reap_node); 893 894 node = next_node(node, node_online_map); 895 if (unlikely(node >= MAX_NUMNODES)) 896 node = first_node(node_online_map); 897 __this_cpu_write(slab_reap_node, node); 898 } 899 900 #else 901 #define init_reap_node(cpu) do { } while (0) 902 #define next_reap_node(void) do { } while (0) 903 #endif 904 905 /* 906 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz 907 * via the workqueue/eventd. 908 * Add the CPU number into the expiration time to minimize the possibility of 909 * the CPUs getting into lockstep and contending for the global cache chain 910 * lock. 911 */ 912 static void __cpuinit start_cpu_timer(int cpu) 913 { 914 struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu); 915 916 /* 917 * When this gets called from do_initcalls via cpucache_init(), 918 * init_workqueues() has already run, so keventd will be setup 919 * at that time. 920 */ 921 if (keventd_up() && reap_work->work.func == NULL) { 922 init_reap_node(cpu); 923 INIT_DELAYED_WORK_DEFERRABLE(reap_work, cache_reap); 924 schedule_delayed_work_on(cpu, reap_work, 925 __round_jiffies_relative(HZ, cpu)); 926 } 927 } 928 929 static struct array_cache *alloc_arraycache(int node, int entries, 930 int batchcount, gfp_t gfp) 931 { 932 int memsize = sizeof(void *) * entries + sizeof(struct array_cache); 933 struct array_cache *nc = NULL; 934 935 nc = kmalloc_node(memsize, gfp, node); 936 /* 937 * The array_cache structures contain pointers to free object. 938 * However, when such objects are allocated or transferred to another 939 * cache the pointers are not cleared and they could be counted as 940 * valid references during a kmemleak scan. Therefore, kmemleak must 941 * not scan such objects. 942 */ 943 kmemleak_no_scan(nc); 944 if (nc) { 945 nc->avail = 0; 946 nc->limit = entries; 947 nc->batchcount = batchcount; 948 nc->touched = 0; 949 spin_lock_init(&nc->lock); 950 } 951 return nc; 952 } 953 954 /* 955 * Transfer objects in one arraycache to another. 956 * Locking must be handled by the caller. 957 * 958 * Return the number of entries transferred. 959 */ 960 static int transfer_objects(struct array_cache *to, 961 struct array_cache *from, unsigned int max) 962 { 963 /* Figure out how many entries to transfer */ 964 int nr = min3(from->avail, max, to->limit - to->avail); 965 966 if (!nr) 967 return 0; 968 969 memcpy(to->entry + to->avail, from->entry + from->avail -nr, 970 sizeof(void *) *nr); 971 972 from->avail -= nr; 973 to->avail += nr; 974 return nr; 975 } 976 977 #ifndef CONFIG_NUMA 978 979 #define drain_alien_cache(cachep, alien) do { } while (0) 980 #define reap_alien(cachep, l3) do { } while (0) 981 982 static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) 983 { 984 return (struct array_cache **)BAD_ALIEN_MAGIC; 985 } 986 987 static inline void free_alien_cache(struct array_cache **ac_ptr) 988 { 989 } 990 991 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 992 { 993 return 0; 994 } 995 996 static inline void *alternate_node_alloc(struct kmem_cache *cachep, 997 gfp_t flags) 998 { 999 return NULL; 1000 } 1001 1002 static inline void *____cache_alloc_node(struct kmem_cache *cachep, 1003 gfp_t flags, int nodeid) 1004 { 1005 return NULL; 1006 } 1007 1008 #else /* CONFIG_NUMA */ 1009 1010 static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int); 1011 static void *alternate_node_alloc(struct kmem_cache *, gfp_t); 1012 1013 static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) 1014 { 1015 struct array_cache **ac_ptr; 1016 int memsize = sizeof(void *) * nr_node_ids; 1017 int i; 1018 1019 if (limit > 1) 1020 limit = 12; 1021 ac_ptr = kzalloc_node(memsize, gfp, node); 1022 if (ac_ptr) { 1023 for_each_node(i) { 1024 if (i == node || !node_online(i)) 1025 continue; 1026 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp); 1027 if (!ac_ptr[i]) { 1028 for (i--; i >= 0; i--) 1029 kfree(ac_ptr[i]); 1030 kfree(ac_ptr); 1031 return NULL; 1032 } 1033 } 1034 } 1035 return ac_ptr; 1036 } 1037 1038 static void free_alien_cache(struct array_cache **ac_ptr) 1039 { 1040 int i; 1041 1042 if (!ac_ptr) 1043 return; 1044 for_each_node(i) 1045 kfree(ac_ptr[i]); 1046 kfree(ac_ptr); 1047 } 1048 1049 static void __drain_alien_cache(struct kmem_cache *cachep, 1050 struct array_cache *ac, int node) 1051 { 1052 struct kmem_list3 *rl3 = cachep->nodelists[node]; 1053 1054 if (ac->avail) { 1055 spin_lock(&rl3->list_lock); 1056 /* 1057 * Stuff objects into the remote nodes shared array first. 1058 * That way we could avoid the overhead of putting the objects 1059 * into the free lists and getting them back later. 1060 */ 1061 if (rl3->shared) 1062 transfer_objects(rl3->shared, ac, ac->limit); 1063 1064 free_block(cachep, ac->entry, ac->avail, node); 1065 ac->avail = 0; 1066 spin_unlock(&rl3->list_lock); 1067 } 1068 } 1069 1070 /* 1071 * Called from cache_reap() to regularly drain alien caches round robin. 1072 */ 1073 static void reap_alien(struct kmem_cache *cachep, struct kmem_list3 *l3) 1074 { 1075 int node = __this_cpu_read(slab_reap_node); 1076 1077 if (l3->alien) { 1078 struct array_cache *ac = l3->alien[node]; 1079 1080 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { 1081 __drain_alien_cache(cachep, ac, node); 1082 spin_unlock_irq(&ac->lock); 1083 } 1084 } 1085 } 1086 1087 static void drain_alien_cache(struct kmem_cache *cachep, 1088 struct array_cache **alien) 1089 { 1090 int i = 0; 1091 struct array_cache *ac; 1092 unsigned long flags; 1093 1094 for_each_online_node(i) { 1095 ac = alien[i]; 1096 if (ac) { 1097 spin_lock_irqsave(&ac->lock, flags); 1098 __drain_alien_cache(cachep, ac, i); 1099 spin_unlock_irqrestore(&ac->lock, flags); 1100 } 1101 } 1102 } 1103 1104 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 1105 { 1106 struct slab *slabp = virt_to_slab(objp); 1107 int nodeid = slabp->nodeid; 1108 struct kmem_list3 *l3; 1109 struct array_cache *alien = NULL; 1110 int node; 1111 1112 node = numa_mem_id(); 1113 1114 /* 1115 * Make sure we are not freeing a object from another node to the array 1116 * cache on this cpu. 1117 */ 1118 if (likely(slabp->nodeid == node)) 1119 return 0; 1120 1121 l3 = cachep->nodelists[node]; 1122 STATS_INC_NODEFREES(cachep); 1123 if (l3->alien && l3->alien[nodeid]) { 1124 alien = l3->alien[nodeid]; 1125 spin_lock(&alien->lock); 1126 if (unlikely(alien->avail == alien->limit)) { 1127 STATS_INC_ACOVERFLOW(cachep); 1128 __drain_alien_cache(cachep, alien, nodeid); 1129 } 1130 alien->entry[alien->avail++] = objp; 1131 spin_unlock(&alien->lock); 1132 } else { 1133 spin_lock(&(cachep->nodelists[nodeid])->list_lock); 1134 free_block(cachep, &objp, 1, nodeid); 1135 spin_unlock(&(cachep->nodelists[nodeid])->list_lock); 1136 } 1137 return 1; 1138 } 1139 #endif 1140 1141 /* 1142 * Allocates and initializes nodelists for a node on each slab cache, used for 1143 * either memory or cpu hotplug. If memory is being hot-added, the kmem_list3 1144 * will be allocated off-node since memory is not yet online for the new node. 1145 * When hotplugging memory or a cpu, existing nodelists are not replaced if 1146 * already in use. 1147 * 1148 * Must hold cache_chain_mutex. 1149 */ 1150 static int init_cache_nodelists_node(int node) 1151 { 1152 struct kmem_cache *cachep; 1153 struct kmem_list3 *l3; 1154 const int memsize = sizeof(struct kmem_list3); 1155 1156 list_for_each_entry(cachep, &cache_chain, next) { 1157 /* 1158 * Set up the size64 kmemlist for cpu before we can 1159 * begin anything. Make sure some other cpu on this 1160 * node has not already allocated this 1161 */ 1162 if (!cachep->nodelists[node]) { 1163 l3 = kmalloc_node(memsize, GFP_KERNEL, node); 1164 if (!l3) 1165 return -ENOMEM; 1166 kmem_list3_init(l3); 1167 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 1168 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 1169 1170 /* 1171 * The l3s don't come and go as CPUs come and 1172 * go. cache_chain_mutex is sufficient 1173 * protection here. 1174 */ 1175 cachep->nodelists[node] = l3; 1176 } 1177 1178 spin_lock_irq(&cachep->nodelists[node]->list_lock); 1179 cachep->nodelists[node]->free_limit = 1180 (1 + nr_cpus_node(node)) * 1181 cachep->batchcount + cachep->num; 1182 spin_unlock_irq(&cachep->nodelists[node]->list_lock); 1183 } 1184 return 0; 1185 } 1186 1187 static void __cpuinit cpuup_canceled(long cpu) 1188 { 1189 struct kmem_cache *cachep; 1190 struct kmem_list3 *l3 = NULL; 1191 int node = cpu_to_mem(cpu); 1192 const struct cpumask *mask = cpumask_of_node(node); 1193 1194 list_for_each_entry(cachep, &cache_chain, next) { 1195 struct array_cache *nc; 1196 struct array_cache *shared; 1197 struct array_cache **alien; 1198 1199 /* cpu is dead; no one can alloc from it. */ 1200 nc = cachep->array[cpu]; 1201 cachep->array[cpu] = NULL; 1202 l3 = cachep->nodelists[node]; 1203 1204 if (!l3) 1205 goto free_array_cache; 1206 1207 spin_lock_irq(&l3->list_lock); 1208 1209 /* Free limit for this kmem_list3 */ 1210 l3->free_limit -= cachep->batchcount; 1211 if (nc) 1212 free_block(cachep, nc->entry, nc->avail, node); 1213 1214 if (!cpumask_empty(mask)) { 1215 spin_unlock_irq(&l3->list_lock); 1216 goto free_array_cache; 1217 } 1218 1219 shared = l3->shared; 1220 if (shared) { 1221 free_block(cachep, shared->entry, 1222 shared->avail, node); 1223 l3->shared = NULL; 1224 } 1225 1226 alien = l3->alien; 1227 l3->alien = NULL; 1228 1229 spin_unlock_irq(&l3->list_lock); 1230 1231 kfree(shared); 1232 if (alien) { 1233 drain_alien_cache(cachep, alien); 1234 free_alien_cache(alien); 1235 } 1236 free_array_cache: 1237 kfree(nc); 1238 } 1239 /* 1240 * In the previous loop, all the objects were freed to 1241 * the respective cache's slabs, now we can go ahead and 1242 * shrink each nodelist to its limit. 1243 */ 1244 list_for_each_entry(cachep, &cache_chain, next) { 1245 l3 = cachep->nodelists[node]; 1246 if (!l3) 1247 continue; 1248 drain_freelist(cachep, l3, l3->free_objects); 1249 } 1250 } 1251 1252 static int __cpuinit cpuup_prepare(long cpu) 1253 { 1254 struct kmem_cache *cachep; 1255 struct kmem_list3 *l3 = NULL; 1256 int node = cpu_to_mem(cpu); 1257 int err; 1258 1259 /* 1260 * We need to do this right in the beginning since 1261 * alloc_arraycache's are going to use this list. 1262 * kmalloc_node allows us to add the slab to the right 1263 * kmem_list3 and not this cpu's kmem_list3 1264 */ 1265 err = init_cache_nodelists_node(node); 1266 if (err < 0) 1267 goto bad; 1268 1269 /* 1270 * Now we can go ahead with allocating the shared arrays and 1271 * array caches 1272 */ 1273 list_for_each_entry(cachep, &cache_chain, next) { 1274 struct array_cache *nc; 1275 struct array_cache *shared = NULL; 1276 struct array_cache **alien = NULL; 1277 1278 nc = alloc_arraycache(node, cachep->limit, 1279 cachep->batchcount, GFP_KERNEL); 1280 if (!nc) 1281 goto bad; 1282 if (cachep->shared) { 1283 shared = alloc_arraycache(node, 1284 cachep->shared * cachep->batchcount, 1285 0xbaadf00d, GFP_KERNEL); 1286 if (!shared) { 1287 kfree(nc); 1288 goto bad; 1289 } 1290 } 1291 if (use_alien_caches) { 1292 alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL); 1293 if (!alien) { 1294 kfree(shared); 1295 kfree(nc); 1296 goto bad; 1297 } 1298 } 1299 cachep->array[cpu] = nc; 1300 l3 = cachep->nodelists[node]; 1301 BUG_ON(!l3); 1302 1303 spin_lock_irq(&l3->list_lock); 1304 if (!l3->shared) { 1305 /* 1306 * We are serialised from CPU_DEAD or 1307 * CPU_UP_CANCELLED by the cpucontrol lock 1308 */ 1309 l3->shared = shared; 1310 shared = NULL; 1311 } 1312 #ifdef CONFIG_NUMA 1313 if (!l3->alien) { 1314 l3->alien = alien; 1315 alien = NULL; 1316 } 1317 #endif 1318 spin_unlock_irq(&l3->list_lock); 1319 kfree(shared); 1320 free_alien_cache(alien); 1321 if (cachep->flags & SLAB_DEBUG_OBJECTS) 1322 slab_set_debugobj_lock_classes_node(cachep, node); 1323 } 1324 init_node_lock_keys(node); 1325 1326 return 0; 1327 bad: 1328 cpuup_canceled(cpu); 1329 return -ENOMEM; 1330 } 1331 1332 static int __cpuinit cpuup_callback(struct notifier_block *nfb, 1333 unsigned long action, void *hcpu) 1334 { 1335 long cpu = (long)hcpu; 1336 int err = 0; 1337 1338 switch (action) { 1339 case CPU_UP_PREPARE: 1340 case CPU_UP_PREPARE_FROZEN: 1341 mutex_lock(&cache_chain_mutex); 1342 err = cpuup_prepare(cpu); 1343 mutex_unlock(&cache_chain_mutex); 1344 break; 1345 case CPU_ONLINE: 1346 case CPU_ONLINE_FROZEN: 1347 start_cpu_timer(cpu); 1348 break; 1349 #ifdef CONFIG_HOTPLUG_CPU 1350 case CPU_DOWN_PREPARE: 1351 case CPU_DOWN_PREPARE_FROZEN: 1352 /* 1353 * Shutdown cache reaper. Note that the cache_chain_mutex is 1354 * held so that if cache_reap() is invoked it cannot do 1355 * anything expensive but will only modify reap_work 1356 * and reschedule the timer. 1357 */ 1358 cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu)); 1359 /* Now the cache_reaper is guaranteed to be not running. */ 1360 per_cpu(slab_reap_work, cpu).work.func = NULL; 1361 break; 1362 case CPU_DOWN_FAILED: 1363 case CPU_DOWN_FAILED_FROZEN: 1364 start_cpu_timer(cpu); 1365 break; 1366 case CPU_DEAD: 1367 case CPU_DEAD_FROZEN: 1368 /* 1369 * Even if all the cpus of a node are down, we don't free the 1370 * kmem_list3 of any cache. This to avoid a race between 1371 * cpu_down, and a kmalloc allocation from another cpu for 1372 * memory from the node of the cpu going down. The list3 1373 * structure is usually allocated from kmem_cache_create() and 1374 * gets destroyed at kmem_cache_destroy(). 1375 */ 1376 /* fall through */ 1377 #endif 1378 case CPU_UP_CANCELED: 1379 case CPU_UP_CANCELED_FROZEN: 1380 mutex_lock(&cache_chain_mutex); 1381 cpuup_canceled(cpu); 1382 mutex_unlock(&cache_chain_mutex); 1383 break; 1384 } 1385 return notifier_from_errno(err); 1386 } 1387 1388 static struct notifier_block __cpuinitdata cpucache_notifier = { 1389 &cpuup_callback, NULL, 0 1390 }; 1391 1392 #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) 1393 /* 1394 * Drains freelist for a node on each slab cache, used for memory hot-remove. 1395 * Returns -EBUSY if all objects cannot be drained so that the node is not 1396 * removed. 1397 * 1398 * Must hold cache_chain_mutex. 1399 */ 1400 static int __meminit drain_cache_nodelists_node(int node) 1401 { 1402 struct kmem_cache *cachep; 1403 int ret = 0; 1404 1405 list_for_each_entry(cachep, &cache_chain, next) { 1406 struct kmem_list3 *l3; 1407 1408 l3 = cachep->nodelists[node]; 1409 if (!l3) 1410 continue; 1411 1412 drain_freelist(cachep, l3, l3->free_objects); 1413 1414 if (!list_empty(&l3->slabs_full) || 1415 !list_empty(&l3->slabs_partial)) { 1416 ret = -EBUSY; 1417 break; 1418 } 1419 } 1420 return ret; 1421 } 1422 1423 static int __meminit slab_memory_callback(struct notifier_block *self, 1424 unsigned long action, void *arg) 1425 { 1426 struct memory_notify *mnb = arg; 1427 int ret = 0; 1428 int nid; 1429 1430 nid = mnb->status_change_nid; 1431 if (nid < 0) 1432 goto out; 1433 1434 switch (action) { 1435 case MEM_GOING_ONLINE: 1436 mutex_lock(&cache_chain_mutex); 1437 ret = init_cache_nodelists_node(nid); 1438 mutex_unlock(&cache_chain_mutex); 1439 break; 1440 case MEM_GOING_OFFLINE: 1441 mutex_lock(&cache_chain_mutex); 1442 ret = drain_cache_nodelists_node(nid); 1443 mutex_unlock(&cache_chain_mutex); 1444 break; 1445 case MEM_ONLINE: 1446 case MEM_OFFLINE: 1447 case MEM_CANCEL_ONLINE: 1448 case MEM_CANCEL_OFFLINE: 1449 break; 1450 } 1451 out: 1452 return notifier_from_errno(ret); 1453 } 1454 #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */ 1455 1456 /* 1457 * swap the static kmem_list3 with kmalloced memory 1458 */ 1459 static void __init init_list(struct kmem_cache *cachep, struct kmem_list3 *list, 1460 int nodeid) 1461 { 1462 struct kmem_list3 *ptr; 1463 1464 ptr = kmalloc_node(sizeof(struct kmem_list3), GFP_NOWAIT, nodeid); 1465 BUG_ON(!ptr); 1466 1467 memcpy(ptr, list, sizeof(struct kmem_list3)); 1468 /* 1469 * Do not assume that spinlocks can be initialized via memcpy: 1470 */ 1471 spin_lock_init(&ptr->list_lock); 1472 1473 MAKE_ALL_LISTS(cachep, ptr, nodeid); 1474 cachep->nodelists[nodeid] = ptr; 1475 } 1476 1477 /* 1478 * For setting up all the kmem_list3s for cache whose buffer_size is same as 1479 * size of kmem_list3. 1480 */ 1481 static void __init set_up_list3s(struct kmem_cache *cachep, int index) 1482 { 1483 int node; 1484 1485 for_each_online_node(node) { 1486 cachep->nodelists[node] = &initkmem_list3[index + node]; 1487 cachep->nodelists[node]->next_reap = jiffies + 1488 REAPTIMEOUT_LIST3 + 1489 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 1490 } 1491 } 1492 1493 /* 1494 * Initialisation. Called after the page allocator have been initialised and 1495 * before smp_init(). 1496 */ 1497 void __init kmem_cache_init(void) 1498 { 1499 size_t left_over; 1500 struct cache_sizes *sizes; 1501 struct cache_names *names; 1502 int i; 1503 int order; 1504 int node; 1505 1506 if (num_possible_nodes() == 1) 1507 use_alien_caches = 0; 1508 1509 for (i = 0; i < NUM_INIT_LISTS; i++) { 1510 kmem_list3_init(&initkmem_list3[i]); 1511 if (i < MAX_NUMNODES) 1512 cache_cache.nodelists[i] = NULL; 1513 } 1514 set_up_list3s(&cache_cache, CACHE_CACHE); 1515 1516 /* 1517 * Fragmentation resistance on low memory - only use bigger 1518 * page orders on machines with more than 32MB of memory if 1519 * not overridden on the command line. 1520 */ 1521 if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT) 1522 slab_max_order = SLAB_MAX_ORDER_HI; 1523 1524 /* Bootstrap is tricky, because several objects are allocated 1525 * from caches that do not exist yet: 1526 * 1) initialize the cache_cache cache: it contains the struct 1527 * kmem_cache structures of all caches, except cache_cache itself: 1528 * cache_cache is statically allocated. 1529 * Initially an __init data area is used for the head array and the 1530 * kmem_list3 structures, it's replaced with a kmalloc allocated 1531 * array at the end of the bootstrap. 1532 * 2) Create the first kmalloc cache. 1533 * The struct kmem_cache for the new cache is allocated normally. 1534 * An __init data area is used for the head array. 1535 * 3) Create the remaining kmalloc caches, with minimally sized 1536 * head arrays. 1537 * 4) Replace the __init data head arrays for cache_cache and the first 1538 * kmalloc cache with kmalloc allocated arrays. 1539 * 5) Replace the __init data for kmem_list3 for cache_cache and 1540 * the other cache's with kmalloc allocated memory. 1541 * 6) Resize the head arrays of the kmalloc caches to their final sizes. 1542 */ 1543 1544 node = numa_mem_id(); 1545 1546 /* 1) create the cache_cache */ 1547 INIT_LIST_HEAD(&cache_chain); 1548 list_add(&cache_cache.next, &cache_chain); 1549 cache_cache.colour_off = cache_line_size(); 1550 cache_cache.array[smp_processor_id()] = &initarray_cache.cache; 1551 cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; 1552 1553 /* 1554 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids 1555 */ 1556 cache_cache.buffer_size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + 1557 nr_node_ids * sizeof(struct kmem_list3 *); 1558 #if DEBUG 1559 cache_cache.obj_size = cache_cache.buffer_size; 1560 #endif 1561 cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, 1562 cache_line_size()); 1563 cache_cache.reciprocal_buffer_size = 1564 reciprocal_value(cache_cache.buffer_size); 1565 1566 for (order = 0; order < MAX_ORDER; order++) { 1567 cache_estimate(order, cache_cache.buffer_size, 1568 cache_line_size(), 0, &left_over, &cache_cache.num); 1569 if (cache_cache.num) 1570 break; 1571 } 1572 BUG_ON(!cache_cache.num); 1573 cache_cache.gfporder = order; 1574 cache_cache.colour = left_over / cache_cache.colour_off; 1575 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) + 1576 sizeof(struct slab), cache_line_size()); 1577 1578 /* 2+3) create the kmalloc caches */ 1579 sizes = malloc_sizes; 1580 names = cache_names; 1581 1582 /* 1583 * Initialize the caches that provide memory for the array cache and the 1584 * kmem_list3 structures first. Without this, further allocations will 1585 * bug. 1586 */ 1587 1588 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, 1589 sizes[INDEX_AC].cs_size, 1590 ARCH_KMALLOC_MINALIGN, 1591 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1592 NULL); 1593 1594 if (INDEX_AC != INDEX_L3) { 1595 sizes[INDEX_L3].cs_cachep = 1596 kmem_cache_create(names[INDEX_L3].name, 1597 sizes[INDEX_L3].cs_size, 1598 ARCH_KMALLOC_MINALIGN, 1599 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1600 NULL); 1601 } 1602 1603 slab_early_init = 0; 1604 1605 while (sizes->cs_size != ULONG_MAX) { 1606 /* 1607 * For performance, all the general caches are L1 aligned. 1608 * This should be particularly beneficial on SMP boxes, as it 1609 * eliminates "false sharing". 1610 * Note for systems short on memory removing the alignment will 1611 * allow tighter packing of the smaller caches. 1612 */ 1613 if (!sizes->cs_cachep) { 1614 sizes->cs_cachep = kmem_cache_create(names->name, 1615 sizes->cs_size, 1616 ARCH_KMALLOC_MINALIGN, 1617 ARCH_KMALLOC_FLAGS|SLAB_PANIC, 1618 NULL); 1619 } 1620 #ifdef CONFIG_ZONE_DMA 1621 sizes->cs_dmacachep = kmem_cache_create( 1622 names->name_dma, 1623 sizes->cs_size, 1624 ARCH_KMALLOC_MINALIGN, 1625 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| 1626 SLAB_PANIC, 1627 NULL); 1628 #endif 1629 sizes++; 1630 names++; 1631 } 1632 /* 4) Replace the bootstrap head arrays */ 1633 { 1634 struct array_cache *ptr; 1635 1636 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1637 1638 BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache); 1639 memcpy(ptr, cpu_cache_get(&cache_cache), 1640 sizeof(struct arraycache_init)); 1641 /* 1642 * Do not assume that spinlocks can be initialized via memcpy: 1643 */ 1644 spin_lock_init(&ptr->lock); 1645 1646 cache_cache.array[smp_processor_id()] = ptr; 1647 1648 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1649 1650 BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) 1651 != &initarray_generic.cache); 1652 memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), 1653 sizeof(struct arraycache_init)); 1654 /* 1655 * Do not assume that spinlocks can be initialized via memcpy: 1656 */ 1657 spin_lock_init(&ptr->lock); 1658 1659 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = 1660 ptr; 1661 } 1662 /* 5) Replace the bootstrap kmem_list3's */ 1663 { 1664 int nid; 1665 1666 for_each_online_node(nid) { 1667 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE + nid], nid); 1668 1669 init_list(malloc_sizes[INDEX_AC].cs_cachep, 1670 &initkmem_list3[SIZE_AC + nid], nid); 1671 1672 if (INDEX_AC != INDEX_L3) { 1673 init_list(malloc_sizes[INDEX_L3].cs_cachep, 1674 &initkmem_list3[SIZE_L3 + nid], nid); 1675 } 1676 } 1677 } 1678 1679 g_cpucache_up = EARLY; 1680 } 1681 1682 void __init kmem_cache_init_late(void) 1683 { 1684 struct kmem_cache *cachep; 1685 1686 g_cpucache_up = LATE; 1687 1688 /* Annotate slab for lockdep -- annotate the malloc caches */ 1689 init_lock_keys(); 1690 1691 /* 6) resize the head arrays to their final sizes */ 1692 mutex_lock(&cache_chain_mutex); 1693 list_for_each_entry(cachep, &cache_chain, next) 1694 if (enable_cpucache(cachep, GFP_NOWAIT)) 1695 BUG(); 1696 mutex_unlock(&cache_chain_mutex); 1697 1698 /* Done! */ 1699 g_cpucache_up = FULL; 1700 1701 /* 1702 * Register a cpu startup notifier callback that initializes 1703 * cpu_cache_get for all new cpus 1704 */ 1705 register_cpu_notifier(&cpucache_notifier); 1706 1707 #ifdef CONFIG_NUMA 1708 /* 1709 * Register a memory hotplug callback that initializes and frees 1710 * nodelists. 1711 */ 1712 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 1713 #endif 1714 1715 /* 1716 * The reap timers are started later, with a module init call: That part 1717 * of the kernel is not yet operational. 1718 */ 1719 } 1720 1721 static int __init cpucache_init(void) 1722 { 1723 int cpu; 1724 1725 /* 1726 * Register the timers that return unneeded pages to the page allocator 1727 */ 1728 for_each_online_cpu(cpu) 1729 start_cpu_timer(cpu); 1730 return 0; 1731 } 1732 __initcall(cpucache_init); 1733 1734 /* 1735 * Interface to system's page allocator. No need to hold the cache-lock. 1736 * 1737 * If we requested dmaable memory, we will get it. Even if we 1738 * did not request dmaable memory, we might get it, but that 1739 * would be relatively rare and ignorable. 1740 */ 1741 static void *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) 1742 { 1743 struct page *page; 1744 int nr_pages; 1745 int i; 1746 1747 #ifndef CONFIG_MMU 1748 /* 1749 * Nommu uses slab's for process anonymous memory allocations, and thus 1750 * requires __GFP_COMP to properly refcount higher order allocations 1751 */ 1752 flags |= __GFP_COMP; 1753 #endif 1754 1755 flags |= cachep->gfpflags; 1756 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1757 flags |= __GFP_RECLAIMABLE; 1758 1759 page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder); 1760 if (!page) 1761 return NULL; 1762 1763 nr_pages = (1 << cachep->gfporder); 1764 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1765 add_zone_page_state(page_zone(page), 1766 NR_SLAB_RECLAIMABLE, nr_pages); 1767 else 1768 add_zone_page_state(page_zone(page), 1769 NR_SLAB_UNRECLAIMABLE, nr_pages); 1770 for (i = 0; i < nr_pages; i++) 1771 __SetPageSlab(page + i); 1772 1773 if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) { 1774 kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid); 1775 1776 if (cachep->ctor) 1777 kmemcheck_mark_uninitialized_pages(page, nr_pages); 1778 else 1779 kmemcheck_mark_unallocated_pages(page, nr_pages); 1780 } 1781 1782 return page_address(page); 1783 } 1784 1785 /* 1786 * Interface to system's page release. 1787 */ 1788 static void kmem_freepages(struct kmem_cache *cachep, void *addr) 1789 { 1790 unsigned long i = (1 << cachep->gfporder); 1791 struct page *page = virt_to_page(addr); 1792 const unsigned long nr_freed = i; 1793 1794 kmemcheck_free_shadow(page, cachep->gfporder); 1795 1796 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1797 sub_zone_page_state(page_zone(page), 1798 NR_SLAB_RECLAIMABLE, nr_freed); 1799 else 1800 sub_zone_page_state(page_zone(page), 1801 NR_SLAB_UNRECLAIMABLE, nr_freed); 1802 while (i--) { 1803 BUG_ON(!PageSlab(page)); 1804 __ClearPageSlab(page); 1805 page++; 1806 } 1807 if (current->reclaim_state) 1808 current->reclaim_state->reclaimed_slab += nr_freed; 1809 free_pages((unsigned long)addr, cachep->gfporder); 1810 } 1811 1812 static void kmem_rcu_free(struct rcu_head *head) 1813 { 1814 struct slab_rcu *slab_rcu = (struct slab_rcu *)head; 1815 struct kmem_cache *cachep = slab_rcu->cachep; 1816 1817 kmem_freepages(cachep, slab_rcu->addr); 1818 if (OFF_SLAB(cachep)) 1819 kmem_cache_free(cachep->slabp_cache, slab_rcu); 1820 } 1821 1822 #if DEBUG 1823 1824 #ifdef CONFIG_DEBUG_PAGEALLOC 1825 static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, 1826 unsigned long caller) 1827 { 1828 int size = obj_size(cachep); 1829 1830 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; 1831 1832 if (size < 5 * sizeof(unsigned long)) 1833 return; 1834 1835 *addr++ = 0x12345678; 1836 *addr++ = caller; 1837 *addr++ = smp_processor_id(); 1838 size -= 3 * sizeof(unsigned long); 1839 { 1840 unsigned long *sptr = &caller; 1841 unsigned long svalue; 1842 1843 while (!kstack_end(sptr)) { 1844 svalue = *sptr++; 1845 if (kernel_text_address(svalue)) { 1846 *addr++ = svalue; 1847 size -= sizeof(unsigned long); 1848 if (size <= sizeof(unsigned long)) 1849 break; 1850 } 1851 } 1852 1853 } 1854 *addr++ = 0x87654321; 1855 } 1856 #endif 1857 1858 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) 1859 { 1860 int size = obj_size(cachep); 1861 addr = &((char *)addr)[obj_offset(cachep)]; 1862 1863 memset(addr, val, size); 1864 *(unsigned char *)(addr + size - 1) = POISON_END; 1865 } 1866 1867 static void dump_line(char *data, int offset, int limit) 1868 { 1869 int i; 1870 unsigned char error = 0; 1871 int bad_count = 0; 1872 1873 printk(KERN_ERR "%03x: ", offset); 1874 for (i = 0; i < limit; i++) { 1875 if (data[offset + i] != POISON_FREE) { 1876 error = data[offset + i]; 1877 bad_count++; 1878 } 1879 } 1880 print_hex_dump(KERN_CONT, "", 0, 16, 1, 1881 &data[offset], limit, 1); 1882 1883 if (bad_count == 1) { 1884 error ^= POISON_FREE; 1885 if (!(error & (error - 1))) { 1886 printk(KERN_ERR "Single bit error detected. Probably " 1887 "bad RAM.\n"); 1888 #ifdef CONFIG_X86 1889 printk(KERN_ERR "Run memtest86+ or a similar memory " 1890 "test tool.\n"); 1891 #else 1892 printk(KERN_ERR "Run a memory test tool.\n"); 1893 #endif 1894 } 1895 } 1896 } 1897 #endif 1898 1899 #if DEBUG 1900 1901 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) 1902 { 1903 int i, size; 1904 char *realobj; 1905 1906 if (cachep->flags & SLAB_RED_ZONE) { 1907 printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n", 1908 *dbg_redzone1(cachep, objp), 1909 *dbg_redzone2(cachep, objp)); 1910 } 1911 1912 if (cachep->flags & SLAB_STORE_USER) { 1913 printk(KERN_ERR "Last user: [<%p>]", 1914 *dbg_userword(cachep, objp)); 1915 print_symbol("(%s)", 1916 (unsigned long)*dbg_userword(cachep, objp)); 1917 printk("\n"); 1918 } 1919 realobj = (char *)objp + obj_offset(cachep); 1920 size = obj_size(cachep); 1921 for (i = 0; i < size && lines; i += 16, lines--) { 1922 int limit; 1923 limit = 16; 1924 if (i + limit > size) 1925 limit = size - i; 1926 dump_line(realobj, i, limit); 1927 } 1928 } 1929 1930 static void check_poison_obj(struct kmem_cache *cachep, void *objp) 1931 { 1932 char *realobj; 1933 int size, i; 1934 int lines = 0; 1935 1936 realobj = (char *)objp + obj_offset(cachep); 1937 size = obj_size(cachep); 1938 1939 for (i = 0; i < size; i++) { 1940 char exp = POISON_FREE; 1941 if (i == size - 1) 1942 exp = POISON_END; 1943 if (realobj[i] != exp) { 1944 int limit; 1945 /* Mismatch ! */ 1946 /* Print header */ 1947 if (lines == 0) { 1948 printk(KERN_ERR 1949 "Slab corruption (%s): %s start=%p, len=%d\n", 1950 print_tainted(), cachep->name, realobj, size); 1951 print_objinfo(cachep, objp, 0); 1952 } 1953 /* Hexdump the affected line */ 1954 i = (i / 16) * 16; 1955 limit = 16; 1956 if (i + limit > size) 1957 limit = size - i; 1958 dump_line(realobj, i, limit); 1959 i += 16; 1960 lines++; 1961 /* Limit to 5 lines */ 1962 if (lines > 5) 1963 break; 1964 } 1965 } 1966 if (lines != 0) { 1967 /* Print some data about the neighboring objects, if they 1968 * exist: 1969 */ 1970 struct slab *slabp = virt_to_slab(objp); 1971 unsigned int objnr; 1972 1973 objnr = obj_to_index(cachep, slabp, objp); 1974 if (objnr) { 1975 objp = index_to_obj(cachep, slabp, objnr - 1); 1976 realobj = (char *)objp + obj_offset(cachep); 1977 printk(KERN_ERR "Prev obj: start=%p, len=%d\n", 1978 realobj, size); 1979 print_objinfo(cachep, objp, 2); 1980 } 1981 if (objnr + 1 < cachep->num) { 1982 objp = index_to_obj(cachep, slabp, objnr + 1); 1983 realobj = (char *)objp + obj_offset(cachep); 1984 printk(KERN_ERR "Next obj: start=%p, len=%d\n", 1985 realobj, size); 1986 print_objinfo(cachep, objp, 2); 1987 } 1988 } 1989 } 1990 #endif 1991 1992 #if DEBUG 1993 static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp) 1994 { 1995 int i; 1996 for (i = 0; i < cachep->num; i++) { 1997 void *objp = index_to_obj(cachep, slabp, i); 1998 1999 if (cachep->flags & SLAB_POISON) { 2000 #ifdef CONFIG_DEBUG_PAGEALLOC 2001 if (cachep->buffer_size % PAGE_SIZE == 0 && 2002 OFF_SLAB(cachep)) 2003 kernel_map_pages(virt_to_page(objp), 2004 cachep->buffer_size / PAGE_SIZE, 1); 2005 else 2006 check_poison_obj(cachep, objp); 2007 #else 2008 check_poison_obj(cachep, objp); 2009 #endif 2010 } 2011 if (cachep->flags & SLAB_RED_ZONE) { 2012 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 2013 slab_error(cachep, "start of a freed object " 2014 "was overwritten"); 2015 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 2016 slab_error(cachep, "end of a freed object " 2017 "was overwritten"); 2018 } 2019 } 2020 } 2021 #else 2022 static void slab_destroy_debugcheck(struct kmem_cache *cachep, struct slab *slabp) 2023 { 2024 } 2025 #endif 2026 2027 /** 2028 * slab_destroy - destroy and release all objects in a slab 2029 * @cachep: cache pointer being destroyed 2030 * @slabp: slab pointer being destroyed 2031 * 2032 * Destroy all the objs in a slab, and release the mem back to the system. 2033 * Before calling the slab must have been unlinked from the cache. The 2034 * cache-lock is not held/needed. 2035 */ 2036 static void slab_destroy(struct kmem_cache *cachep, struct slab *slabp) 2037 { 2038 void *addr = slabp->s_mem - slabp->colouroff; 2039 2040 slab_destroy_debugcheck(cachep, slabp); 2041 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { 2042 struct slab_rcu *slab_rcu; 2043 2044 slab_rcu = (struct slab_rcu *)slabp; 2045 slab_rcu->cachep = cachep; 2046 slab_rcu->addr = addr; 2047 call_rcu(&slab_rcu->head, kmem_rcu_free); 2048 } else { 2049 kmem_freepages(cachep, addr); 2050 if (OFF_SLAB(cachep)) 2051 kmem_cache_free(cachep->slabp_cache, slabp); 2052 } 2053 } 2054 2055 static void __kmem_cache_destroy(struct kmem_cache *cachep) 2056 { 2057 int i; 2058 struct kmem_list3 *l3; 2059 2060 for_each_online_cpu(i) 2061 kfree(cachep->array[i]); 2062 2063 /* NUMA: free the list3 structures */ 2064 for_each_online_node(i) { 2065 l3 = cachep->nodelists[i]; 2066 if (l3) { 2067 kfree(l3->shared); 2068 free_alien_cache(l3->alien); 2069 kfree(l3); 2070 } 2071 } 2072 kmem_cache_free(&cache_cache, cachep); 2073 } 2074 2075 2076 /** 2077 * calculate_slab_order - calculate size (page order) of slabs 2078 * @cachep: pointer to the cache that is being created 2079 * @size: size of objects to be created in this cache. 2080 * @align: required alignment for the objects. 2081 * @flags: slab allocation flags 2082 * 2083 * Also calculates the number of objects per slab. 2084 * 2085 * This could be made much more intelligent. For now, try to avoid using 2086 * high order pages for slabs. When the gfp() functions are more friendly 2087 * towards high-order requests, this should be changed. 2088 */ 2089 static size_t calculate_slab_order(struct kmem_cache *cachep, 2090 size_t size, size_t align, unsigned long flags) 2091 { 2092 unsigned long offslab_limit; 2093 size_t left_over = 0; 2094 int gfporder; 2095 2096 for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) { 2097 unsigned int num; 2098 size_t remainder; 2099 2100 cache_estimate(gfporder, size, align, flags, &remainder, &num); 2101 if (!num) 2102 continue; 2103 2104 if (flags & CFLGS_OFF_SLAB) { 2105 /* 2106 * Max number of objs-per-slab for caches which 2107 * use off-slab slabs. Needed to avoid a possible 2108 * looping condition in cache_grow(). 2109 */ 2110 offslab_limit = size - sizeof(struct slab); 2111 offslab_limit /= sizeof(kmem_bufctl_t); 2112 2113 if (num > offslab_limit) 2114 break; 2115 } 2116 2117 /* Found something acceptable - save it away */ 2118 cachep->num = num; 2119 cachep->gfporder = gfporder; 2120 left_over = remainder; 2121 2122 /* 2123 * A VFS-reclaimable slab tends to have most allocations 2124 * as GFP_NOFS and we really don't want to have to be allocating 2125 * higher-order pages when we are unable to shrink dcache. 2126 */ 2127 if (flags & SLAB_RECLAIM_ACCOUNT) 2128 break; 2129 2130 /* 2131 * Large number of objects is good, but very large slabs are 2132 * currently bad for the gfp()s. 2133 */ 2134 if (gfporder >= slab_max_order) 2135 break; 2136 2137 /* 2138 * Acceptable internal fragmentation? 2139 */ 2140 if (left_over * 8 <= (PAGE_SIZE << gfporder)) 2141 break; 2142 } 2143 return left_over; 2144 } 2145 2146 static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) 2147 { 2148 if (g_cpucache_up == FULL) 2149 return enable_cpucache(cachep, gfp); 2150 2151 if (g_cpucache_up == NONE) { 2152 /* 2153 * Note: the first kmem_cache_create must create the cache 2154 * that's used by kmalloc(24), otherwise the creation of 2155 * further caches will BUG(). 2156 */ 2157 cachep->array[smp_processor_id()] = &initarray_generic.cache; 2158 2159 /* 2160 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is 2161 * the first cache, then we need to set up all its list3s, 2162 * otherwise the creation of further caches will BUG(). 2163 */ 2164 set_up_list3s(cachep, SIZE_AC); 2165 if (INDEX_AC == INDEX_L3) 2166 g_cpucache_up = PARTIAL_L3; 2167 else 2168 g_cpucache_up = PARTIAL_AC; 2169 } else { 2170 cachep->array[smp_processor_id()] = 2171 kmalloc(sizeof(struct arraycache_init), gfp); 2172 2173 if (g_cpucache_up == PARTIAL_AC) { 2174 set_up_list3s(cachep, SIZE_L3); 2175 g_cpucache_up = PARTIAL_L3; 2176 } else { 2177 int node; 2178 for_each_online_node(node) { 2179 cachep->nodelists[node] = 2180 kmalloc_node(sizeof(struct kmem_list3), 2181 gfp, node); 2182 BUG_ON(!cachep->nodelists[node]); 2183 kmem_list3_init(cachep->nodelists[node]); 2184 } 2185 } 2186 } 2187 cachep->nodelists[numa_mem_id()]->next_reap = 2188 jiffies + REAPTIMEOUT_LIST3 + 2189 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 2190 2191 cpu_cache_get(cachep)->avail = 0; 2192 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; 2193 cpu_cache_get(cachep)->batchcount = 1; 2194 cpu_cache_get(cachep)->touched = 0; 2195 cachep->batchcount = 1; 2196 cachep->limit = BOOT_CPUCACHE_ENTRIES; 2197 return 0; 2198 } 2199 2200 /** 2201 * kmem_cache_create - Create a cache. 2202 * @name: A string which is used in /proc/slabinfo to identify this cache. 2203 * @size: The size of objects to be created in this cache. 2204 * @align: The required alignment for the objects. 2205 * @flags: SLAB flags 2206 * @ctor: A constructor for the objects. 2207 * 2208 * Returns a ptr to the cache on success, NULL on failure. 2209 * Cannot be called within a int, but can be interrupted. 2210 * The @ctor is run when new pages are allocated by the cache. 2211 * 2212 * @name must be valid until the cache is destroyed. This implies that 2213 * the module calling this has to destroy the cache before getting unloaded. 2214 * 2215 * The flags are 2216 * 2217 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 2218 * to catch references to uninitialised memory. 2219 * 2220 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 2221 * for buffer overruns. 2222 * 2223 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 2224 * cacheline. This can be beneficial if you're counting cycles as closely 2225 * as davem. 2226 */ 2227 struct kmem_cache * 2228 kmem_cache_create (const char *name, size_t size, size_t align, 2229 unsigned long flags, void (*ctor)(void *)) 2230 { 2231 size_t left_over, slab_size, ralign; 2232 struct kmem_cache *cachep = NULL, *pc; 2233 gfp_t gfp; 2234 2235 /* 2236 * Sanity checks... these are all serious usage bugs. 2237 */ 2238 if (!name || in_interrupt() || (size < BYTES_PER_WORD) || 2239 size > KMALLOC_MAX_SIZE) { 2240 printk(KERN_ERR "%s: Early error in slab %s\n", __func__, 2241 name); 2242 BUG(); 2243 } 2244 2245 /* 2246 * We use cache_chain_mutex to ensure a consistent view of 2247 * cpu_online_mask as well. Please see cpuup_callback 2248 */ 2249 if (slab_is_available()) { 2250 get_online_cpus(); 2251 mutex_lock(&cache_chain_mutex); 2252 } 2253 2254 list_for_each_entry(pc, &cache_chain, next) { 2255 char tmp; 2256 int res; 2257 2258 /* 2259 * This happens when the module gets unloaded and doesn't 2260 * destroy its slab cache and no-one else reuses the vmalloc 2261 * area of the module. Print a warning. 2262 */ 2263 res = probe_kernel_address(pc->name, tmp); 2264 if (res) { 2265 printk(KERN_ERR 2266 "SLAB: cache with size %d has lost its name\n", 2267 pc->buffer_size); 2268 continue; 2269 } 2270 2271 if (!strcmp(pc->name, name)) { 2272 printk(KERN_ERR 2273 "kmem_cache_create: duplicate cache %s\n", name); 2274 dump_stack(); 2275 goto oops; 2276 } 2277 } 2278 2279 #if DEBUG 2280 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 2281 #if FORCED_DEBUG 2282 /* 2283 * Enable redzoning and last user accounting, except for caches with 2284 * large objects, if the increased size would increase the object size 2285 * above the next power of two: caches with object sizes just above a 2286 * power of two have a significant amount of internal fragmentation. 2287 */ 2288 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN + 2289 2 * sizeof(unsigned long long))) 2290 flags |= SLAB_RED_ZONE | SLAB_STORE_USER; 2291 if (!(flags & SLAB_DESTROY_BY_RCU)) 2292 flags |= SLAB_POISON; 2293 #endif 2294 if (flags & SLAB_DESTROY_BY_RCU) 2295 BUG_ON(flags & SLAB_POISON); 2296 #endif 2297 /* 2298 * Always checks flags, a caller might be expecting debug support which 2299 * isn't available. 2300 */ 2301 BUG_ON(flags & ~CREATE_MASK); 2302 2303 /* 2304 * Check that size is in terms of words. This is needed to avoid 2305 * unaligned accesses for some archs when redzoning is used, and makes 2306 * sure any on-slab bufctl's are also correctly aligned. 2307 */ 2308 if (size & (BYTES_PER_WORD - 1)) { 2309 size += (BYTES_PER_WORD - 1); 2310 size &= ~(BYTES_PER_WORD - 1); 2311 } 2312 2313 /* calculate the final buffer alignment: */ 2314 2315 /* 1) arch recommendation: can be overridden for debug */ 2316 if (flags & SLAB_HWCACHE_ALIGN) { 2317 /* 2318 * Default alignment: as specified by the arch code. Except if 2319 * an object is really small, then squeeze multiple objects into 2320 * one cacheline. 2321 */ 2322 ralign = cache_line_size(); 2323 while (size <= ralign / 2) 2324 ralign /= 2; 2325 } else { 2326 ralign = BYTES_PER_WORD; 2327 } 2328 2329 /* 2330 * Redzoning and user store require word alignment or possibly larger. 2331 * Note this will be overridden by architecture or caller mandated 2332 * alignment if either is greater than BYTES_PER_WORD. 2333 */ 2334 if (flags & SLAB_STORE_USER) 2335 ralign = BYTES_PER_WORD; 2336 2337 if (flags & SLAB_RED_ZONE) { 2338 ralign = REDZONE_ALIGN; 2339 /* If redzoning, ensure that the second redzone is suitably 2340 * aligned, by adjusting the object size accordingly. */ 2341 size += REDZONE_ALIGN - 1; 2342 size &= ~(REDZONE_ALIGN - 1); 2343 } 2344 2345 /* 2) arch mandated alignment */ 2346 if (ralign < ARCH_SLAB_MINALIGN) { 2347 ralign = ARCH_SLAB_MINALIGN; 2348 } 2349 /* 3) caller mandated alignment */ 2350 if (ralign < align) { 2351 ralign = align; 2352 } 2353 /* disable debug if necessary */ 2354 if (ralign > __alignof__(unsigned long long)) 2355 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2356 /* 2357 * 4) Store it. 2358 */ 2359 align = ralign; 2360 2361 if (slab_is_available()) 2362 gfp = GFP_KERNEL; 2363 else 2364 gfp = GFP_NOWAIT; 2365 2366 /* Get cache's description obj. */ 2367 cachep = kmem_cache_zalloc(&cache_cache, gfp); 2368 if (!cachep) 2369 goto oops; 2370 2371 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; 2372 #if DEBUG 2373 cachep->obj_size = size; 2374 2375 /* 2376 * Both debugging options require word-alignment which is calculated 2377 * into align above. 2378 */ 2379 if (flags & SLAB_RED_ZONE) { 2380 /* add space for red zone words */ 2381 cachep->obj_offset += sizeof(unsigned long long); 2382 size += 2 * sizeof(unsigned long long); 2383 } 2384 if (flags & SLAB_STORE_USER) { 2385 /* user store requires one word storage behind the end of 2386 * the real object. But if the second red zone needs to be 2387 * aligned to 64 bits, we must allow that much space. 2388 */ 2389 if (flags & SLAB_RED_ZONE) 2390 size += REDZONE_ALIGN; 2391 else 2392 size += BYTES_PER_WORD; 2393 } 2394 #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) 2395 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size 2396 && cachep->obj_size > cache_line_size() && ALIGN(size, align) < PAGE_SIZE) { 2397 cachep->obj_offset += PAGE_SIZE - ALIGN(size, align); 2398 size = PAGE_SIZE; 2399 } 2400 #endif 2401 #endif 2402 2403 /* 2404 * Determine if the slab management is 'on' or 'off' slab. 2405 * (bootstrapping cannot cope with offslab caches so don't do 2406 * it too early on. Always use on-slab management when 2407 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak) 2408 */ 2409 if ((size >= (PAGE_SIZE >> 3)) && !slab_early_init && 2410 !(flags & SLAB_NOLEAKTRACE)) 2411 /* 2412 * Size is large, assume best to place the slab management obj 2413 * off-slab (should allow better packing of objs). 2414 */ 2415 flags |= CFLGS_OFF_SLAB; 2416 2417 size = ALIGN(size, align); 2418 2419 left_over = calculate_slab_order(cachep, size, align, flags); 2420 2421 if (!cachep->num) { 2422 printk(KERN_ERR 2423 "kmem_cache_create: couldn't create cache %s.\n", name); 2424 kmem_cache_free(&cache_cache, cachep); 2425 cachep = NULL; 2426 goto oops; 2427 } 2428 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t) 2429 + sizeof(struct slab), align); 2430 2431 /* 2432 * If the slab has been placed off-slab, and we have enough space then 2433 * move it on-slab. This is at the expense of any extra colouring. 2434 */ 2435 if (flags & CFLGS_OFF_SLAB && left_over >= slab_size) { 2436 flags &= ~CFLGS_OFF_SLAB; 2437 left_over -= slab_size; 2438 } 2439 2440 if (flags & CFLGS_OFF_SLAB) { 2441 /* really off slab. No need for manual alignment */ 2442 slab_size = 2443 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab); 2444 2445 #ifdef CONFIG_PAGE_POISONING 2446 /* If we're going to use the generic kernel_map_pages() 2447 * poisoning, then it's going to smash the contents of 2448 * the redzone and userword anyhow, so switch them off. 2449 */ 2450 if (size % PAGE_SIZE == 0 && flags & SLAB_POISON) 2451 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2452 #endif 2453 } 2454 2455 cachep->colour_off = cache_line_size(); 2456 /* Offset must be a multiple of the alignment. */ 2457 if (cachep->colour_off < align) 2458 cachep->colour_off = align; 2459 cachep->colour = left_over / cachep->colour_off; 2460 cachep->slab_size = slab_size; 2461 cachep->flags = flags; 2462 cachep->gfpflags = 0; 2463 if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) 2464 cachep->gfpflags |= GFP_DMA; 2465 cachep->buffer_size = size; 2466 cachep->reciprocal_buffer_size = reciprocal_value(size); 2467 2468 if (flags & CFLGS_OFF_SLAB) { 2469 cachep->slabp_cache = kmem_find_general_cachep(slab_size, 0u); 2470 /* 2471 * This is a possibility for one of the malloc_sizes caches. 2472 * But since we go off slab only for object size greater than 2473 * PAGE_SIZE/8, and malloc_sizes gets created in ascending order, 2474 * this should not happen at all. 2475 * But leave a BUG_ON for some lucky dude. 2476 */ 2477 BUG_ON(ZERO_OR_NULL_PTR(cachep->slabp_cache)); 2478 } 2479 cachep->ctor = ctor; 2480 cachep->name = name; 2481 2482 if (setup_cpu_cache(cachep, gfp)) { 2483 __kmem_cache_destroy(cachep); 2484 cachep = NULL; 2485 goto oops; 2486 } 2487 2488 if (flags & SLAB_DEBUG_OBJECTS) { 2489 /* 2490 * Would deadlock through slab_destroy()->call_rcu()-> 2491 * debug_object_activate()->kmem_cache_alloc(). 2492 */ 2493 WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU); 2494 2495 slab_set_debugobj_lock_classes(cachep); 2496 } 2497 2498 /* cache setup completed, link it into the list */ 2499 list_add(&cachep->next, &cache_chain); 2500 oops: 2501 if (!cachep && (flags & SLAB_PANIC)) 2502 panic("kmem_cache_create(): failed to create slab `%s'\n", 2503 name); 2504 if (slab_is_available()) { 2505 mutex_unlock(&cache_chain_mutex); 2506 put_online_cpus(); 2507 } 2508 return cachep; 2509 } 2510 EXPORT_SYMBOL(kmem_cache_create); 2511 2512 #if DEBUG 2513 static void check_irq_off(void) 2514 { 2515 BUG_ON(!irqs_disabled()); 2516 } 2517 2518 static void check_irq_on(void) 2519 { 2520 BUG_ON(irqs_disabled()); 2521 } 2522 2523 static void check_spinlock_acquired(struct kmem_cache *cachep) 2524 { 2525 #ifdef CONFIG_SMP 2526 check_irq_off(); 2527 assert_spin_locked(&cachep->nodelists[numa_mem_id()]->list_lock); 2528 #endif 2529 } 2530 2531 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) 2532 { 2533 #ifdef CONFIG_SMP 2534 check_irq_off(); 2535 assert_spin_locked(&cachep->nodelists[node]->list_lock); 2536 #endif 2537 } 2538 2539 #else 2540 #define check_irq_off() do { } while(0) 2541 #define check_irq_on() do { } while(0) 2542 #define check_spinlock_acquired(x) do { } while(0) 2543 #define check_spinlock_acquired_node(x, y) do { } while(0) 2544 #endif 2545 2546 static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, 2547 struct array_cache *ac, 2548 int force, int node); 2549 2550 static void do_drain(void *arg) 2551 { 2552 struct kmem_cache *cachep = arg; 2553 struct array_cache *ac; 2554 int node = numa_mem_id(); 2555 2556 check_irq_off(); 2557 ac = cpu_cache_get(cachep); 2558 spin_lock(&cachep->nodelists[node]->list_lock); 2559 free_block(cachep, ac->entry, ac->avail, node); 2560 spin_unlock(&cachep->nodelists[node]->list_lock); 2561 ac->avail = 0; 2562 } 2563 2564 static void drain_cpu_caches(struct kmem_cache *cachep) 2565 { 2566 struct kmem_list3 *l3; 2567 int node; 2568 2569 on_each_cpu(do_drain, cachep, 1); 2570 check_irq_on(); 2571 for_each_online_node(node) { 2572 l3 = cachep->nodelists[node]; 2573 if (l3 && l3->alien) 2574 drain_alien_cache(cachep, l3->alien); 2575 } 2576 2577 for_each_online_node(node) { 2578 l3 = cachep->nodelists[node]; 2579 if (l3) 2580 drain_array(cachep, l3, l3->shared, 1, node); 2581 } 2582 } 2583 2584 /* 2585 * Remove slabs from the list of free slabs. 2586 * Specify the number of slabs to drain in tofree. 2587 * 2588 * Returns the actual number of slabs released. 2589 */ 2590 static int drain_freelist(struct kmem_cache *cache, 2591 struct kmem_list3 *l3, int tofree) 2592 { 2593 struct list_head *p; 2594 int nr_freed; 2595 struct slab *slabp; 2596 2597 nr_freed = 0; 2598 while (nr_freed < tofree && !list_empty(&l3->slabs_free)) { 2599 2600 spin_lock_irq(&l3->list_lock); 2601 p = l3->slabs_free.prev; 2602 if (p == &l3->slabs_free) { 2603 spin_unlock_irq(&l3->list_lock); 2604 goto out; 2605 } 2606 2607 slabp = list_entry(p, struct slab, list); 2608 #if DEBUG 2609 BUG_ON(slabp->inuse); 2610 #endif 2611 list_del(&slabp->list); 2612 /* 2613 * Safe to drop the lock. The slab is no longer linked 2614 * to the cache. 2615 */ 2616 l3->free_objects -= cache->num; 2617 spin_unlock_irq(&l3->list_lock); 2618 slab_destroy(cache, slabp); 2619 nr_freed++; 2620 } 2621 out: 2622 return nr_freed; 2623 } 2624 2625 /* Called with cache_chain_mutex held to protect against cpu hotplug */ 2626 static int __cache_shrink(struct kmem_cache *cachep) 2627 { 2628 int ret = 0, i = 0; 2629 struct kmem_list3 *l3; 2630 2631 drain_cpu_caches(cachep); 2632 2633 check_irq_on(); 2634 for_each_online_node(i) { 2635 l3 = cachep->nodelists[i]; 2636 if (!l3) 2637 continue; 2638 2639 drain_freelist(cachep, l3, l3->free_objects); 2640 2641 ret += !list_empty(&l3->slabs_full) || 2642 !list_empty(&l3->slabs_partial); 2643 } 2644 return (ret ? 1 : 0); 2645 } 2646 2647 /** 2648 * kmem_cache_shrink - Shrink a cache. 2649 * @cachep: The cache to shrink. 2650 * 2651 * Releases as many slabs as possible for a cache. 2652 * To help debugging, a zero exit status indicates all slabs were released. 2653 */ 2654 int kmem_cache_shrink(struct kmem_cache *cachep) 2655 { 2656 int ret; 2657 BUG_ON(!cachep || in_interrupt()); 2658 2659 get_online_cpus(); 2660 mutex_lock(&cache_chain_mutex); 2661 ret = __cache_shrink(cachep); 2662 mutex_unlock(&cache_chain_mutex); 2663 put_online_cpus(); 2664 return ret; 2665 } 2666 EXPORT_SYMBOL(kmem_cache_shrink); 2667 2668 /** 2669 * kmem_cache_destroy - delete a cache 2670 * @cachep: the cache to destroy 2671 * 2672 * Remove a &struct kmem_cache object from the slab cache. 2673 * 2674 * It is expected this function will be called by a module when it is 2675 * unloaded. This will remove the cache completely, and avoid a duplicate 2676 * cache being allocated each time a module is loaded and unloaded, if the 2677 * module doesn't have persistent in-kernel storage across loads and unloads. 2678 * 2679 * The cache must be empty before calling this function. 2680 * 2681 * The caller must guarantee that no one will allocate memory from the cache 2682 * during the kmem_cache_destroy(). 2683 */ 2684 void kmem_cache_destroy(struct kmem_cache *cachep) 2685 { 2686 BUG_ON(!cachep || in_interrupt()); 2687 2688 /* Find the cache in the chain of caches. */ 2689 get_online_cpus(); 2690 mutex_lock(&cache_chain_mutex); 2691 /* 2692 * the chain is never empty, cache_cache is never destroyed 2693 */ 2694 list_del(&cachep->next); 2695 if (__cache_shrink(cachep)) { 2696 slab_error(cachep, "Can't free all objects"); 2697 list_add(&cachep->next, &cache_chain); 2698 mutex_unlock(&cache_chain_mutex); 2699 put_online_cpus(); 2700 return; 2701 } 2702 2703 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) 2704 rcu_barrier(); 2705 2706 __kmem_cache_destroy(cachep); 2707 mutex_unlock(&cache_chain_mutex); 2708 put_online_cpus(); 2709 } 2710 EXPORT_SYMBOL(kmem_cache_destroy); 2711 2712 /* 2713 * Get the memory for a slab management obj. 2714 * For a slab cache when the slab descriptor is off-slab, slab descriptors 2715 * always come from malloc_sizes caches. The slab descriptor cannot 2716 * come from the same cache which is getting created because, 2717 * when we are searching for an appropriate cache for these 2718 * descriptors in kmem_cache_create, we search through the malloc_sizes array. 2719 * If we are creating a malloc_sizes cache here it would not be visible to 2720 * kmem_find_general_cachep till the initialization is complete. 2721 * Hence we cannot have slabp_cache same as the original cache. 2722 */ 2723 static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp, 2724 int colour_off, gfp_t local_flags, 2725 int nodeid) 2726 { 2727 struct slab *slabp; 2728 2729 if (OFF_SLAB(cachep)) { 2730 /* Slab management obj is off-slab. */ 2731 slabp = kmem_cache_alloc_node(cachep->slabp_cache, 2732 local_flags, nodeid); 2733 /* 2734 * If the first object in the slab is leaked (it's allocated 2735 * but no one has a reference to it), we want to make sure 2736 * kmemleak does not treat the ->s_mem pointer as a reference 2737 * to the object. Otherwise we will not report the leak. 2738 */ 2739 kmemleak_scan_area(&slabp->list, sizeof(struct list_head), 2740 local_flags); 2741 if (!slabp) 2742 return NULL; 2743 } else { 2744 slabp = objp + colour_off; 2745 colour_off += cachep->slab_size; 2746 } 2747 slabp->inuse = 0; 2748 slabp->colouroff = colour_off; 2749 slabp->s_mem = objp + colour_off; 2750 slabp->nodeid = nodeid; 2751 slabp->free = 0; 2752 return slabp; 2753 } 2754 2755 static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) 2756 { 2757 return (kmem_bufctl_t *) (slabp + 1); 2758 } 2759 2760 static void cache_init_objs(struct kmem_cache *cachep, 2761 struct slab *slabp) 2762 { 2763 int i; 2764 2765 for (i = 0; i < cachep->num; i++) { 2766 void *objp = index_to_obj(cachep, slabp, i); 2767 #if DEBUG 2768 /* need to poison the objs? */ 2769 if (cachep->flags & SLAB_POISON) 2770 poison_obj(cachep, objp, POISON_FREE); 2771 if (cachep->flags & SLAB_STORE_USER) 2772 *dbg_userword(cachep, objp) = NULL; 2773 2774 if (cachep->flags & SLAB_RED_ZONE) { 2775 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2776 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2777 } 2778 /* 2779 * Constructors are not allowed to allocate memory from the same 2780 * cache which they are a constructor for. Otherwise, deadlock. 2781 * They must also be threaded. 2782 */ 2783 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) 2784 cachep->ctor(objp + obj_offset(cachep)); 2785 2786 if (cachep->flags & SLAB_RED_ZONE) { 2787 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 2788 slab_error(cachep, "constructor overwrote the" 2789 " end of an object"); 2790 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 2791 slab_error(cachep, "constructor overwrote the" 2792 " start of an object"); 2793 } 2794 if ((cachep->buffer_size % PAGE_SIZE) == 0 && 2795 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) 2796 kernel_map_pages(virt_to_page(objp), 2797 cachep->buffer_size / PAGE_SIZE, 0); 2798 #else 2799 if (cachep->ctor) 2800 cachep->ctor(objp); 2801 #endif 2802 slab_bufctl(slabp)[i] = i + 1; 2803 } 2804 slab_bufctl(slabp)[i - 1] = BUFCTL_END; 2805 } 2806 2807 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) 2808 { 2809 if (CONFIG_ZONE_DMA_FLAG) { 2810 if (flags & GFP_DMA) 2811 BUG_ON(!(cachep->gfpflags & GFP_DMA)); 2812 else 2813 BUG_ON(cachep->gfpflags & GFP_DMA); 2814 } 2815 } 2816 2817 static void *slab_get_obj(struct kmem_cache *cachep, struct slab *slabp, 2818 int nodeid) 2819 { 2820 void *objp = index_to_obj(cachep, slabp, slabp->free); 2821 kmem_bufctl_t next; 2822 2823 slabp->inuse++; 2824 next = slab_bufctl(slabp)[slabp->free]; 2825 #if DEBUG 2826 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; 2827 WARN_ON(slabp->nodeid != nodeid); 2828 #endif 2829 slabp->free = next; 2830 2831 return objp; 2832 } 2833 2834 static void slab_put_obj(struct kmem_cache *cachep, struct slab *slabp, 2835 void *objp, int nodeid) 2836 { 2837 unsigned int objnr = obj_to_index(cachep, slabp, objp); 2838 2839 #if DEBUG 2840 /* Verify that the slab belongs to the intended node */ 2841 WARN_ON(slabp->nodeid != nodeid); 2842 2843 if (slab_bufctl(slabp)[objnr] + 1 <= SLAB_LIMIT + 1) { 2844 printk(KERN_ERR "slab: double free detected in cache " 2845 "'%s', objp %p\n", cachep->name, objp); 2846 BUG(); 2847 } 2848 #endif 2849 slab_bufctl(slabp)[objnr] = slabp->free; 2850 slabp->free = objnr; 2851 slabp->inuse--; 2852 } 2853 2854 /* 2855 * Map pages beginning at addr to the given cache and slab. This is required 2856 * for the slab allocator to be able to lookup the cache and slab of a 2857 * virtual address for kfree, ksize, and slab debugging. 2858 */ 2859 static void slab_map_pages(struct kmem_cache *cache, struct slab *slab, 2860 void *addr) 2861 { 2862 int nr_pages; 2863 struct page *page; 2864 2865 page = virt_to_page(addr); 2866 2867 nr_pages = 1; 2868 if (likely(!PageCompound(page))) 2869 nr_pages <<= cache->gfporder; 2870 2871 do { 2872 page_set_cache(page, cache); 2873 page_set_slab(page, slab); 2874 page++; 2875 } while (--nr_pages); 2876 } 2877 2878 /* 2879 * Grow (by 1) the number of slabs within a cache. This is called by 2880 * kmem_cache_alloc() when there are no active objs left in a cache. 2881 */ 2882 static int cache_grow(struct kmem_cache *cachep, 2883 gfp_t flags, int nodeid, void *objp) 2884 { 2885 struct slab *slabp; 2886 size_t offset; 2887 gfp_t local_flags; 2888 struct kmem_list3 *l3; 2889 2890 /* 2891 * Be lazy and only check for valid flags here, keeping it out of the 2892 * critical path in kmem_cache_alloc(). 2893 */ 2894 BUG_ON(flags & GFP_SLAB_BUG_MASK); 2895 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); 2896 2897 /* Take the l3 list lock to change the colour_next on this node */ 2898 check_irq_off(); 2899 l3 = cachep->nodelists[nodeid]; 2900 spin_lock(&l3->list_lock); 2901 2902 /* Get colour for the slab, and cal the next value. */ 2903 offset = l3->colour_next; 2904 l3->colour_next++; 2905 if (l3->colour_next >= cachep->colour) 2906 l3->colour_next = 0; 2907 spin_unlock(&l3->list_lock); 2908 2909 offset *= cachep->colour_off; 2910 2911 if (local_flags & __GFP_WAIT) 2912 local_irq_enable(); 2913 2914 /* 2915 * The test for missing atomic flag is performed here, rather than 2916 * the more obvious place, simply to reduce the critical path length 2917 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they 2918 * will eventually be caught here (where it matters). 2919 */ 2920 kmem_flagcheck(cachep, flags); 2921 2922 /* 2923 * Get mem for the objs. Attempt to allocate a physical page from 2924 * 'nodeid'. 2925 */ 2926 if (!objp) 2927 objp = kmem_getpages(cachep, local_flags, nodeid); 2928 if (!objp) 2929 goto failed; 2930 2931 /* Get slab management. */ 2932 slabp = alloc_slabmgmt(cachep, objp, offset, 2933 local_flags & ~GFP_CONSTRAINT_MASK, nodeid); 2934 if (!slabp) 2935 goto opps1; 2936 2937 slab_map_pages(cachep, slabp, objp); 2938 2939 cache_init_objs(cachep, slabp); 2940 2941 if (local_flags & __GFP_WAIT) 2942 local_irq_disable(); 2943 check_irq_off(); 2944 spin_lock(&l3->list_lock); 2945 2946 /* Make slab active. */ 2947 list_add_tail(&slabp->list, &(l3->slabs_free)); 2948 STATS_INC_GROWN(cachep); 2949 l3->free_objects += cachep->num; 2950 spin_unlock(&l3->list_lock); 2951 return 1; 2952 opps1: 2953 kmem_freepages(cachep, objp); 2954 failed: 2955 if (local_flags & __GFP_WAIT) 2956 local_irq_disable(); 2957 return 0; 2958 } 2959 2960 #if DEBUG 2961 2962 /* 2963 * Perform extra freeing checks: 2964 * - detect bad pointers. 2965 * - POISON/RED_ZONE checking 2966 */ 2967 static void kfree_debugcheck(const void *objp) 2968 { 2969 if (!virt_addr_valid(objp)) { 2970 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", 2971 (unsigned long)objp); 2972 BUG(); 2973 } 2974 } 2975 2976 static inline void verify_redzone_free(struct kmem_cache *cache, void *obj) 2977 { 2978 unsigned long long redzone1, redzone2; 2979 2980 redzone1 = *dbg_redzone1(cache, obj); 2981 redzone2 = *dbg_redzone2(cache, obj); 2982 2983 /* 2984 * Redzone is ok. 2985 */ 2986 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE) 2987 return; 2988 2989 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE) 2990 slab_error(cache, "double free detected"); 2991 else 2992 slab_error(cache, "memory outside object was overwritten"); 2993 2994 printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n", 2995 obj, redzone1, redzone2); 2996 } 2997 2998 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, 2999 void *caller) 3000 { 3001 struct page *page; 3002 unsigned int objnr; 3003 struct slab *slabp; 3004 3005 BUG_ON(virt_to_cache(objp) != cachep); 3006 3007 objp -= obj_offset(cachep); 3008 kfree_debugcheck(objp); 3009 page = virt_to_head_page(objp); 3010 3011 slabp = page_get_slab(page); 3012 3013 if (cachep->flags & SLAB_RED_ZONE) { 3014 verify_redzone_free(cachep, objp); 3015 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 3016 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 3017 } 3018 if (cachep->flags & SLAB_STORE_USER) 3019 *dbg_userword(cachep, objp) = caller; 3020 3021 objnr = obj_to_index(cachep, slabp, objp); 3022 3023 BUG_ON(objnr >= cachep->num); 3024 BUG_ON(objp != index_to_obj(cachep, slabp, objnr)); 3025 3026 #ifdef CONFIG_DEBUG_SLAB_LEAK 3027 slab_bufctl(slabp)[objnr] = BUFCTL_FREE; 3028 #endif 3029 if (cachep->flags & SLAB_POISON) { 3030 #ifdef CONFIG_DEBUG_PAGEALLOC 3031 if ((cachep->buffer_size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { 3032 store_stackinfo(cachep, objp, (unsigned long)caller); 3033 kernel_map_pages(virt_to_page(objp), 3034 cachep->buffer_size / PAGE_SIZE, 0); 3035 } else { 3036 poison_obj(cachep, objp, POISON_FREE); 3037 } 3038 #else 3039 poison_obj(cachep, objp, POISON_FREE); 3040 #endif 3041 } 3042 return objp; 3043 } 3044 3045 static void check_slabp(struct kmem_cache *cachep, struct slab *slabp) 3046 { 3047 kmem_bufctl_t i; 3048 int entries = 0; 3049 3050 /* Check slab's freelist to see if this obj is there. */ 3051 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { 3052 entries++; 3053 if (entries > cachep->num || i >= cachep->num) 3054 goto bad; 3055 } 3056 if (entries != cachep->num - slabp->inuse) { 3057 bad: 3058 printk(KERN_ERR "slab: Internal list corruption detected in " 3059 "cache '%s'(%d), slabp %p(%d). Tainted(%s). Hexdump:\n", 3060 cachep->name, cachep->num, slabp, slabp->inuse, 3061 print_tainted()); 3062 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 16, 1, slabp, 3063 sizeof(*slabp) + cachep->num * sizeof(kmem_bufctl_t), 3064 1); 3065 BUG(); 3066 } 3067 } 3068 #else 3069 #define kfree_debugcheck(x) do { } while(0) 3070 #define cache_free_debugcheck(x,objp,z) (objp) 3071 #define check_slabp(x,y) do { } while(0) 3072 #endif 3073 3074 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) 3075 { 3076 int batchcount; 3077 struct kmem_list3 *l3; 3078 struct array_cache *ac; 3079 int node; 3080 3081 retry: 3082 check_irq_off(); 3083 node = numa_mem_id(); 3084 ac = cpu_cache_get(cachep); 3085 batchcount = ac->batchcount; 3086 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { 3087 /* 3088 * If there was little recent activity on this cache, then 3089 * perform only a partial refill. Otherwise we could generate 3090 * refill bouncing. 3091 */ 3092 batchcount = BATCHREFILL_LIMIT; 3093 } 3094 l3 = cachep->nodelists[node]; 3095 3096 BUG_ON(ac->avail > 0 || !l3); 3097 spin_lock(&l3->list_lock); 3098 3099 /* See if we can refill from the shared array */ 3100 if (l3->shared && transfer_objects(ac, l3->shared, batchcount)) { 3101 l3->shared->touched = 1; 3102 goto alloc_done; 3103 } 3104 3105 while (batchcount > 0) { 3106 struct list_head *entry; 3107 struct slab *slabp; 3108 /* Get slab alloc is to come from. */ 3109 entry = l3->slabs_partial.next; 3110 if (entry == &l3->slabs_partial) { 3111 l3->free_touched = 1; 3112 entry = l3->slabs_free.next; 3113 if (entry == &l3->slabs_free) 3114 goto must_grow; 3115 } 3116 3117 slabp = list_entry(entry, struct slab, list); 3118 check_slabp(cachep, slabp); 3119 check_spinlock_acquired(cachep); 3120 3121 /* 3122 * The slab was either on partial or free list so 3123 * there must be at least one object available for 3124 * allocation. 3125 */ 3126 BUG_ON(slabp->inuse >= cachep->num); 3127 3128 while (slabp->inuse < cachep->num && batchcount--) { 3129 STATS_INC_ALLOCED(cachep); 3130 STATS_INC_ACTIVE(cachep); 3131 STATS_SET_HIGH(cachep); 3132 3133 ac->entry[ac->avail++] = slab_get_obj(cachep, slabp, 3134 node); 3135 } 3136 check_slabp(cachep, slabp); 3137 3138 /* move slabp to correct slabp list: */ 3139 list_del(&slabp->list); 3140 if (slabp->free == BUFCTL_END) 3141 list_add(&slabp->list, &l3->slabs_full); 3142 else 3143 list_add(&slabp->list, &l3->slabs_partial); 3144 } 3145 3146 must_grow: 3147 l3->free_objects -= ac->avail; 3148 alloc_done: 3149 spin_unlock(&l3->list_lock); 3150 3151 if (unlikely(!ac->avail)) { 3152 int x; 3153 x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL); 3154 3155 /* cache_grow can reenable interrupts, then ac could change. */ 3156 ac = cpu_cache_get(cachep); 3157 if (!x && ac->avail == 0) /* no objects in sight? abort */ 3158 return NULL; 3159 3160 if (!ac->avail) /* objects refilled by interrupt? */ 3161 goto retry; 3162 } 3163 ac->touched = 1; 3164 return ac->entry[--ac->avail]; 3165 } 3166 3167 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, 3168 gfp_t flags) 3169 { 3170 might_sleep_if(flags & __GFP_WAIT); 3171 #if DEBUG 3172 kmem_flagcheck(cachep, flags); 3173 #endif 3174 } 3175 3176 #if DEBUG 3177 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, 3178 gfp_t flags, void *objp, void *caller) 3179 { 3180 if (!objp) 3181 return objp; 3182 if (cachep->flags & SLAB_POISON) { 3183 #ifdef CONFIG_DEBUG_PAGEALLOC 3184 if ((cachep->buffer_size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) 3185 kernel_map_pages(virt_to_page(objp), 3186 cachep->buffer_size / PAGE_SIZE, 1); 3187 else 3188 check_poison_obj(cachep, objp); 3189 #else 3190 check_poison_obj(cachep, objp); 3191 #endif 3192 poison_obj(cachep, objp, POISON_INUSE); 3193 } 3194 if (cachep->flags & SLAB_STORE_USER) 3195 *dbg_userword(cachep, objp) = caller; 3196 3197 if (cachep->flags & SLAB_RED_ZONE) { 3198 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || 3199 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { 3200 slab_error(cachep, "double free, or memory outside" 3201 " object was overwritten"); 3202 printk(KERN_ERR 3203 "%p: redzone 1:0x%llx, redzone 2:0x%llx\n", 3204 objp, *dbg_redzone1(cachep, objp), 3205 *dbg_redzone2(cachep, objp)); 3206 } 3207 *dbg_redzone1(cachep, objp) = RED_ACTIVE; 3208 *dbg_redzone2(cachep, objp) = RED_ACTIVE; 3209 } 3210 #ifdef CONFIG_DEBUG_SLAB_LEAK 3211 { 3212 struct slab *slabp; 3213 unsigned objnr; 3214 3215 slabp = page_get_slab(virt_to_head_page(objp)); 3216 objnr = (unsigned)(objp - slabp->s_mem) / cachep->buffer_size; 3217 slab_bufctl(slabp)[objnr] = BUFCTL_ACTIVE; 3218 } 3219 #endif 3220 objp += obj_offset(cachep); 3221 if (cachep->ctor && cachep->flags & SLAB_POISON) 3222 cachep->ctor(objp); 3223 if (ARCH_SLAB_MINALIGN && 3224 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { 3225 printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n", 3226 objp, (int)ARCH_SLAB_MINALIGN); 3227 } 3228 return objp; 3229 } 3230 #else 3231 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp) 3232 #endif 3233 3234 static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) 3235 { 3236 if (cachep == &cache_cache) 3237 return false; 3238 3239 return should_failslab(obj_size(cachep), flags, cachep->flags); 3240 } 3241 3242 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3243 { 3244 void *objp; 3245 struct array_cache *ac; 3246 3247 check_irq_off(); 3248 3249 ac = cpu_cache_get(cachep); 3250 if (likely(ac->avail)) { 3251 STATS_INC_ALLOCHIT(cachep); 3252 ac->touched = 1; 3253 objp = ac->entry[--ac->avail]; 3254 } else { 3255 STATS_INC_ALLOCMISS(cachep); 3256 objp = cache_alloc_refill(cachep, flags); 3257 /* 3258 * the 'ac' may be updated by cache_alloc_refill(), 3259 * and kmemleak_erase() requires its correct value. 3260 */ 3261 ac = cpu_cache_get(cachep); 3262 } 3263 /* 3264 * To avoid a false negative, if an object that is in one of the 3265 * per-CPU caches is leaked, we need to make sure kmemleak doesn't 3266 * treat the array pointers as a reference to the object. 3267 */ 3268 if (objp) 3269 kmemleak_erase(&ac->entry[ac->avail]); 3270 return objp; 3271 } 3272 3273 #ifdef CONFIG_NUMA 3274 /* 3275 * Try allocating on another node if PF_SPREAD_SLAB|PF_MEMPOLICY. 3276 * 3277 * If we are in_interrupt, then process context, including cpusets and 3278 * mempolicy, may not apply and should not be used for allocation policy. 3279 */ 3280 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) 3281 { 3282 int nid_alloc, nid_here; 3283 3284 if (in_interrupt() || (flags & __GFP_THISNODE)) 3285 return NULL; 3286 nid_alloc = nid_here = numa_mem_id(); 3287 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) 3288 nid_alloc = cpuset_slab_spread_node(); 3289 else if (current->mempolicy) 3290 nid_alloc = slab_node(current->mempolicy); 3291 if (nid_alloc != nid_here) 3292 return ____cache_alloc_node(cachep, flags, nid_alloc); 3293 return NULL; 3294 } 3295 3296 /* 3297 * Fallback function if there was no memory available and no objects on a 3298 * certain node and fall back is permitted. First we scan all the 3299 * available nodelists for available objects. If that fails then we 3300 * perform an allocation without specifying a node. This allows the page 3301 * allocator to do its reclaim / fallback magic. We then insert the 3302 * slab into the proper nodelist and then allocate from it. 3303 */ 3304 static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) 3305 { 3306 struct zonelist *zonelist; 3307 gfp_t local_flags; 3308 struct zoneref *z; 3309 struct zone *zone; 3310 enum zone_type high_zoneidx = gfp_zone(flags); 3311 void *obj = NULL; 3312 int nid; 3313 unsigned int cpuset_mems_cookie; 3314 3315 if (flags & __GFP_THISNODE) 3316 return NULL; 3317 3318 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); 3319 3320 retry_cpuset: 3321 cpuset_mems_cookie = get_mems_allowed(); 3322 zonelist = node_zonelist(slab_node(current->mempolicy), flags); 3323 3324 retry: 3325 /* 3326 * Look through allowed nodes for objects available 3327 * from existing per node queues. 3328 */ 3329 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 3330 nid = zone_to_nid(zone); 3331 3332 if (cpuset_zone_allowed_hardwall(zone, flags) && 3333 cache->nodelists[nid] && 3334 cache->nodelists[nid]->free_objects) { 3335 obj = ____cache_alloc_node(cache, 3336 flags | GFP_THISNODE, nid); 3337 if (obj) 3338 break; 3339 } 3340 } 3341 3342 if (!obj) { 3343 /* 3344 * This allocation will be performed within the constraints 3345 * of the current cpuset / memory policy requirements. 3346 * We may trigger various forms of reclaim on the allowed 3347 * set and go into memory reserves if necessary. 3348 */ 3349 if (local_flags & __GFP_WAIT) 3350 local_irq_enable(); 3351 kmem_flagcheck(cache, flags); 3352 obj = kmem_getpages(cache, local_flags, numa_mem_id()); 3353 if (local_flags & __GFP_WAIT) 3354 local_irq_disable(); 3355 if (obj) { 3356 /* 3357 * Insert into the appropriate per node queues 3358 */ 3359 nid = page_to_nid(virt_to_page(obj)); 3360 if (cache_grow(cache, flags, nid, obj)) { 3361 obj = ____cache_alloc_node(cache, 3362 flags | GFP_THISNODE, nid); 3363 if (!obj) 3364 /* 3365 * Another processor may allocate the 3366 * objects in the slab since we are 3367 * not holding any locks. 3368 */ 3369 goto retry; 3370 } else { 3371 /* cache_grow already freed obj */ 3372 obj = NULL; 3373 } 3374 } 3375 } 3376 3377 if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !obj)) 3378 goto retry_cpuset; 3379 return obj; 3380 } 3381 3382 /* 3383 * A interface to enable slab creation on nodeid 3384 */ 3385 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, 3386 int nodeid) 3387 { 3388 struct list_head *entry; 3389 struct slab *slabp; 3390 struct kmem_list3 *l3; 3391 void *obj; 3392 int x; 3393 3394 l3 = cachep->nodelists[nodeid]; 3395 BUG_ON(!l3); 3396 3397 retry: 3398 check_irq_off(); 3399 spin_lock(&l3->list_lock); 3400 entry = l3->slabs_partial.next; 3401 if (entry == &l3->slabs_partial) { 3402 l3->free_touched = 1; 3403 entry = l3->slabs_free.next; 3404 if (entry == &l3->slabs_free) 3405 goto must_grow; 3406 } 3407 3408 slabp = list_entry(entry, struct slab, list); 3409 check_spinlock_acquired_node(cachep, nodeid); 3410 check_slabp(cachep, slabp); 3411 3412 STATS_INC_NODEALLOCS(cachep); 3413 STATS_INC_ACTIVE(cachep); 3414 STATS_SET_HIGH(cachep); 3415 3416 BUG_ON(slabp->inuse == cachep->num); 3417 3418 obj = slab_get_obj(cachep, slabp, nodeid); 3419 check_slabp(cachep, slabp); 3420 l3->free_objects--; 3421 /* move slabp to correct slabp list: */ 3422 list_del(&slabp->list); 3423 3424 if (slabp->free == BUFCTL_END) 3425 list_add(&slabp->list, &l3->slabs_full); 3426 else 3427 list_add(&slabp->list, &l3->slabs_partial); 3428 3429 spin_unlock(&l3->list_lock); 3430 goto done; 3431 3432 must_grow: 3433 spin_unlock(&l3->list_lock); 3434 x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL); 3435 if (x) 3436 goto retry; 3437 3438 return fallback_alloc(cachep, flags); 3439 3440 done: 3441 return obj; 3442 } 3443 3444 /** 3445 * kmem_cache_alloc_node - Allocate an object on the specified node 3446 * @cachep: The cache to allocate from. 3447 * @flags: See kmalloc(). 3448 * @nodeid: node number of the target node. 3449 * @caller: return address of caller, used for debug information 3450 * 3451 * Identical to kmem_cache_alloc but it will allocate memory on the given 3452 * node, which can improve the performance for cpu bound structures. 3453 * 3454 * Fallback to other node is possible if __GFP_THISNODE is not set. 3455 */ 3456 static __always_inline void * 3457 __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, 3458 void *caller) 3459 { 3460 unsigned long save_flags; 3461 void *ptr; 3462 int slab_node = numa_mem_id(); 3463 3464 flags &= gfp_allowed_mask; 3465 3466 lockdep_trace_alloc(flags); 3467 3468 if (slab_should_failslab(cachep, flags)) 3469 return NULL; 3470 3471 cache_alloc_debugcheck_before(cachep, flags); 3472 local_irq_save(save_flags); 3473 3474 if (nodeid == NUMA_NO_NODE) 3475 nodeid = slab_node; 3476 3477 if (unlikely(!cachep->nodelists[nodeid])) { 3478 /* Node not bootstrapped yet */ 3479 ptr = fallback_alloc(cachep, flags); 3480 goto out; 3481 } 3482 3483 if (nodeid == slab_node) { 3484 /* 3485 * Use the locally cached objects if possible. 3486 * However ____cache_alloc does not allow fallback 3487 * to other nodes. It may fail while we still have 3488 * objects on other nodes available. 3489 */ 3490 ptr = ____cache_alloc(cachep, flags); 3491 if (ptr) 3492 goto out; 3493 } 3494 /* ___cache_alloc_node can fall back to other nodes */ 3495 ptr = ____cache_alloc_node(cachep, flags, nodeid); 3496 out: 3497 local_irq_restore(save_flags); 3498 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); 3499 kmemleak_alloc_recursive(ptr, obj_size(cachep), 1, cachep->flags, 3500 flags); 3501 3502 if (likely(ptr)) 3503 kmemcheck_slab_alloc(cachep, flags, ptr, obj_size(cachep)); 3504 3505 if (unlikely((flags & __GFP_ZERO) && ptr)) 3506 memset(ptr, 0, obj_size(cachep)); 3507 3508 return ptr; 3509 } 3510 3511 static __always_inline void * 3512 __do_cache_alloc(struct kmem_cache *cache, gfp_t flags) 3513 { 3514 void *objp; 3515 3516 if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) { 3517 objp = alternate_node_alloc(cache, flags); 3518 if (objp) 3519 goto out; 3520 } 3521 objp = ____cache_alloc(cache, flags); 3522 3523 /* 3524 * We may just have run out of memory on the local node. 3525 * ____cache_alloc_node() knows how to locate memory on other nodes 3526 */ 3527 if (!objp) 3528 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); 3529 3530 out: 3531 return objp; 3532 } 3533 #else 3534 3535 static __always_inline void * 3536 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3537 { 3538 return ____cache_alloc(cachep, flags); 3539 } 3540 3541 #endif /* CONFIG_NUMA */ 3542 3543 static __always_inline void * 3544 __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) 3545 { 3546 unsigned long save_flags; 3547 void *objp; 3548 3549 flags &= gfp_allowed_mask; 3550 3551 lockdep_trace_alloc(flags); 3552 3553 if (slab_should_failslab(cachep, flags)) 3554 return NULL; 3555 3556 cache_alloc_debugcheck_before(cachep, flags); 3557 local_irq_save(save_flags); 3558 objp = __do_cache_alloc(cachep, flags); 3559 local_irq_restore(save_flags); 3560 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); 3561 kmemleak_alloc_recursive(objp, obj_size(cachep), 1, cachep->flags, 3562 flags); 3563 prefetchw(objp); 3564 3565 if (likely(objp)) 3566 kmemcheck_slab_alloc(cachep, flags, objp, obj_size(cachep)); 3567 3568 if (unlikely((flags & __GFP_ZERO) && objp)) 3569 memset(objp, 0, obj_size(cachep)); 3570 3571 return objp; 3572 } 3573 3574 /* 3575 * Caller needs to acquire correct kmem_list's list_lock 3576 */ 3577 static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, 3578 int node) 3579 { 3580 int i; 3581 struct kmem_list3 *l3; 3582 3583 for (i = 0; i < nr_objects; i++) { 3584 void *objp = objpp[i]; 3585 struct slab *slabp; 3586 3587 slabp = virt_to_slab(objp); 3588 l3 = cachep->nodelists[node]; 3589 list_del(&slabp->list); 3590 check_spinlock_acquired_node(cachep, node); 3591 check_slabp(cachep, slabp); 3592 slab_put_obj(cachep, slabp, objp, node); 3593 STATS_DEC_ACTIVE(cachep); 3594 l3->free_objects++; 3595 check_slabp(cachep, slabp); 3596 3597 /* fixup slab chains */ 3598 if (slabp->inuse == 0) { 3599 if (l3->free_objects > l3->free_limit) { 3600 l3->free_objects -= cachep->num; 3601 /* No need to drop any previously held 3602 * lock here, even if we have a off-slab slab 3603 * descriptor it is guaranteed to come from 3604 * a different cache, refer to comments before 3605 * alloc_slabmgmt. 3606 */ 3607 slab_destroy(cachep, slabp); 3608 } else { 3609 list_add(&slabp->list, &l3->slabs_free); 3610 } 3611 } else { 3612 /* Unconditionally move a slab to the end of the 3613 * partial list on free - maximum time for the 3614 * other objects to be freed, too. 3615 */ 3616 list_add_tail(&slabp->list, &l3->slabs_partial); 3617 } 3618 } 3619 } 3620 3621 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) 3622 { 3623 int batchcount; 3624 struct kmem_list3 *l3; 3625 int node = numa_mem_id(); 3626 3627 batchcount = ac->batchcount; 3628 #if DEBUG 3629 BUG_ON(!batchcount || batchcount > ac->avail); 3630 #endif 3631 check_irq_off(); 3632 l3 = cachep->nodelists[node]; 3633 spin_lock(&l3->list_lock); 3634 if (l3->shared) { 3635 struct array_cache *shared_array = l3->shared; 3636 int max = shared_array->limit - shared_array->avail; 3637 if (max) { 3638 if (batchcount > max) 3639 batchcount = max; 3640 memcpy(&(shared_array->entry[shared_array->avail]), 3641 ac->entry, sizeof(void *) * batchcount); 3642 shared_array->avail += batchcount; 3643 goto free_done; 3644 } 3645 } 3646 3647 free_block(cachep, ac->entry, batchcount, node); 3648 free_done: 3649 #if STATS 3650 { 3651 int i = 0; 3652 struct list_head *p; 3653 3654 p = l3->slabs_free.next; 3655 while (p != &(l3->slabs_free)) { 3656 struct slab *slabp; 3657 3658 slabp = list_entry(p, struct slab, list); 3659 BUG_ON(slabp->inuse); 3660 3661 i++; 3662 p = p->next; 3663 } 3664 STATS_SET_FREEABLE(cachep, i); 3665 } 3666 #endif 3667 spin_unlock(&l3->list_lock); 3668 ac->avail -= batchcount; 3669 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); 3670 } 3671 3672 /* 3673 * Release an obj back to its cache. If the obj has a constructed state, it must 3674 * be in this state _before_ it is released. Called with disabled ints. 3675 */ 3676 static inline void __cache_free(struct kmem_cache *cachep, void *objp, 3677 void *caller) 3678 { 3679 struct array_cache *ac = cpu_cache_get(cachep); 3680 3681 check_irq_off(); 3682 kmemleak_free_recursive(objp, cachep->flags); 3683 objp = cache_free_debugcheck(cachep, objp, caller); 3684 3685 kmemcheck_slab_free(cachep, objp, obj_size(cachep)); 3686 3687 /* 3688 * Skip calling cache_free_alien() when the platform is not numa. 3689 * This will avoid cache misses that happen while accessing slabp (which 3690 * is per page memory reference) to get nodeid. Instead use a global 3691 * variable to skip the call, which is mostly likely to be present in 3692 * the cache. 3693 */ 3694 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) 3695 return; 3696 3697 if (likely(ac->avail < ac->limit)) { 3698 STATS_INC_FREEHIT(cachep); 3699 ac->entry[ac->avail++] = objp; 3700 return; 3701 } else { 3702 STATS_INC_FREEMISS(cachep); 3703 cache_flusharray(cachep, ac); 3704 ac->entry[ac->avail++] = objp; 3705 } 3706 } 3707 3708 /** 3709 * kmem_cache_alloc - Allocate an object 3710 * @cachep: The cache to allocate from. 3711 * @flags: See kmalloc(). 3712 * 3713 * Allocate an object from this cache. The flags are only relevant 3714 * if the cache has no available objects. 3715 */ 3716 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3717 { 3718 void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); 3719 3720 trace_kmem_cache_alloc(_RET_IP_, ret, 3721 obj_size(cachep), cachep->buffer_size, flags); 3722 3723 return ret; 3724 } 3725 EXPORT_SYMBOL(kmem_cache_alloc); 3726 3727 #ifdef CONFIG_TRACING 3728 void * 3729 kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags) 3730 { 3731 void *ret; 3732 3733 ret = __cache_alloc(cachep, flags, __builtin_return_address(0)); 3734 3735 trace_kmalloc(_RET_IP_, ret, 3736 size, slab_buffer_size(cachep), flags); 3737 return ret; 3738 } 3739 EXPORT_SYMBOL(kmem_cache_alloc_trace); 3740 #endif 3741 3742 #ifdef CONFIG_NUMA 3743 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 3744 { 3745 void *ret = __cache_alloc_node(cachep, flags, nodeid, 3746 __builtin_return_address(0)); 3747 3748 trace_kmem_cache_alloc_node(_RET_IP_, ret, 3749 obj_size(cachep), cachep->buffer_size, 3750 flags, nodeid); 3751 3752 return ret; 3753 } 3754 EXPORT_SYMBOL(kmem_cache_alloc_node); 3755 3756 #ifdef CONFIG_TRACING 3757 void *kmem_cache_alloc_node_trace(size_t size, 3758 struct kmem_cache *cachep, 3759 gfp_t flags, 3760 int nodeid) 3761 { 3762 void *ret; 3763 3764 ret = __cache_alloc_node(cachep, flags, nodeid, 3765 __builtin_return_address(0)); 3766 trace_kmalloc_node(_RET_IP_, ret, 3767 size, slab_buffer_size(cachep), 3768 flags, nodeid); 3769 return ret; 3770 } 3771 EXPORT_SYMBOL(kmem_cache_alloc_node_trace); 3772 #endif 3773 3774 static __always_inline void * 3775 __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller) 3776 { 3777 struct kmem_cache *cachep; 3778 3779 cachep = kmem_find_general_cachep(size, flags); 3780 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3781 return cachep; 3782 return kmem_cache_alloc_node_trace(size, cachep, flags, node); 3783 } 3784 3785 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) 3786 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3787 { 3788 return __do_kmalloc_node(size, flags, node, 3789 __builtin_return_address(0)); 3790 } 3791 EXPORT_SYMBOL(__kmalloc_node); 3792 3793 void *__kmalloc_node_track_caller(size_t size, gfp_t flags, 3794 int node, unsigned long caller) 3795 { 3796 return __do_kmalloc_node(size, flags, node, (void *)caller); 3797 } 3798 EXPORT_SYMBOL(__kmalloc_node_track_caller); 3799 #else 3800 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3801 { 3802 return __do_kmalloc_node(size, flags, node, NULL); 3803 } 3804 EXPORT_SYMBOL(__kmalloc_node); 3805 #endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */ 3806 #endif /* CONFIG_NUMA */ 3807 3808 /** 3809 * __do_kmalloc - allocate memory 3810 * @size: how many bytes of memory are required. 3811 * @flags: the type of memory to allocate (see kmalloc). 3812 * @caller: function caller for debug tracking of the caller 3813 */ 3814 static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, 3815 void *caller) 3816 { 3817 struct kmem_cache *cachep; 3818 void *ret; 3819 3820 /* If you want to save a few bytes .text space: replace 3821 * __ with kmem_. 3822 * Then kmalloc uses the uninlined functions instead of the inline 3823 * functions. 3824 */ 3825 cachep = __find_general_cachep(size, flags); 3826 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3827 return cachep; 3828 ret = __cache_alloc(cachep, flags, caller); 3829 3830 trace_kmalloc((unsigned long) caller, ret, 3831 size, cachep->buffer_size, flags); 3832 3833 return ret; 3834 } 3835 3836 3837 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) 3838 void *__kmalloc(size_t size, gfp_t flags) 3839 { 3840 return __do_kmalloc(size, flags, __builtin_return_address(0)); 3841 } 3842 EXPORT_SYMBOL(__kmalloc); 3843 3844 void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller) 3845 { 3846 return __do_kmalloc(size, flags, (void *)caller); 3847 } 3848 EXPORT_SYMBOL(__kmalloc_track_caller); 3849 3850 #else 3851 void *__kmalloc(size_t size, gfp_t flags) 3852 { 3853 return __do_kmalloc(size, flags, NULL); 3854 } 3855 EXPORT_SYMBOL(__kmalloc); 3856 #endif 3857 3858 /** 3859 * kmem_cache_free - Deallocate an object 3860 * @cachep: The cache the allocation was from. 3861 * @objp: The previously allocated object. 3862 * 3863 * Free an object which was previously allocated from this 3864 * cache. 3865 */ 3866 void kmem_cache_free(struct kmem_cache *cachep, void *objp) 3867 { 3868 unsigned long flags; 3869 3870 local_irq_save(flags); 3871 debug_check_no_locks_freed(objp, obj_size(cachep)); 3872 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) 3873 debug_check_no_obj_freed(objp, obj_size(cachep)); 3874 __cache_free(cachep, objp, __builtin_return_address(0)); 3875 local_irq_restore(flags); 3876 3877 trace_kmem_cache_free(_RET_IP_, objp); 3878 } 3879 EXPORT_SYMBOL(kmem_cache_free); 3880 3881 /** 3882 * kfree - free previously allocated memory 3883 * @objp: pointer returned by kmalloc. 3884 * 3885 * If @objp is NULL, no operation is performed. 3886 * 3887 * Don't free memory not originally allocated by kmalloc() 3888 * or you will run into trouble. 3889 */ 3890 void kfree(const void *objp) 3891 { 3892 struct kmem_cache *c; 3893 unsigned long flags; 3894 3895 trace_kfree(_RET_IP_, objp); 3896 3897 if (unlikely(ZERO_OR_NULL_PTR(objp))) 3898 return; 3899 local_irq_save(flags); 3900 kfree_debugcheck(objp); 3901 c = virt_to_cache(objp); 3902 debug_check_no_locks_freed(objp, obj_size(c)); 3903 debug_check_no_obj_freed(objp, obj_size(c)); 3904 __cache_free(c, (void *)objp, __builtin_return_address(0)); 3905 local_irq_restore(flags); 3906 } 3907 EXPORT_SYMBOL(kfree); 3908 3909 unsigned int kmem_cache_size(struct kmem_cache *cachep) 3910 { 3911 return obj_size(cachep); 3912 } 3913 EXPORT_SYMBOL(kmem_cache_size); 3914 3915 /* 3916 * This initializes kmem_list3 or resizes various caches for all nodes. 3917 */ 3918 static int alloc_kmemlist(struct kmem_cache *cachep, gfp_t gfp) 3919 { 3920 int node; 3921 struct kmem_list3 *l3; 3922 struct array_cache *new_shared; 3923 struct array_cache **new_alien = NULL; 3924 3925 for_each_online_node(node) { 3926 3927 if (use_alien_caches) { 3928 new_alien = alloc_alien_cache(node, cachep->limit, gfp); 3929 if (!new_alien) 3930 goto fail; 3931 } 3932 3933 new_shared = NULL; 3934 if (cachep->shared) { 3935 new_shared = alloc_arraycache(node, 3936 cachep->shared*cachep->batchcount, 3937 0xbaadf00d, gfp); 3938 if (!new_shared) { 3939 free_alien_cache(new_alien); 3940 goto fail; 3941 } 3942 } 3943 3944 l3 = cachep->nodelists[node]; 3945 if (l3) { 3946 struct array_cache *shared = l3->shared; 3947 3948 spin_lock_irq(&l3->list_lock); 3949 3950 if (shared) 3951 free_block(cachep, shared->entry, 3952 shared->avail, node); 3953 3954 l3->shared = new_shared; 3955 if (!l3->alien) { 3956 l3->alien = new_alien; 3957 new_alien = NULL; 3958 } 3959 l3->free_limit = (1 + nr_cpus_node(node)) * 3960 cachep->batchcount + cachep->num; 3961 spin_unlock_irq(&l3->list_lock); 3962 kfree(shared); 3963 free_alien_cache(new_alien); 3964 continue; 3965 } 3966 l3 = kmalloc_node(sizeof(struct kmem_list3), gfp, node); 3967 if (!l3) { 3968 free_alien_cache(new_alien); 3969 kfree(new_shared); 3970 goto fail; 3971 } 3972 3973 kmem_list3_init(l3); 3974 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 3975 ((unsigned long)cachep) % REAPTIMEOUT_LIST3; 3976 l3->shared = new_shared; 3977 l3->alien = new_alien; 3978 l3->free_limit = (1 + nr_cpus_node(node)) * 3979 cachep->batchcount + cachep->num; 3980 cachep->nodelists[node] = l3; 3981 } 3982 return 0; 3983 3984 fail: 3985 if (!cachep->next.next) { 3986 /* Cache is not active yet. Roll back what we did */ 3987 node--; 3988 while (node >= 0) { 3989 if (cachep->nodelists[node]) { 3990 l3 = cachep->nodelists[node]; 3991 3992 kfree(l3->shared); 3993 free_alien_cache(l3->alien); 3994 kfree(l3); 3995 cachep->nodelists[node] = NULL; 3996 } 3997 node--; 3998 } 3999 } 4000 return -ENOMEM; 4001 } 4002 4003 struct ccupdate_struct { 4004 struct kmem_cache *cachep; 4005 struct array_cache *new[0]; 4006 }; 4007 4008 static void do_ccupdate_local(void *info) 4009 { 4010 struct ccupdate_struct *new = info; 4011 struct array_cache *old; 4012 4013 check_irq_off(); 4014 old = cpu_cache_get(new->cachep); 4015 4016 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()]; 4017 new->new[smp_processor_id()] = old; 4018 } 4019 4020 /* Always called with the cache_chain_mutex held */ 4021 static int do_tune_cpucache(struct kmem_cache *cachep, int limit, 4022 int batchcount, int shared, gfp_t gfp) 4023 { 4024 struct ccupdate_struct *new; 4025 int i; 4026 4027 new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *), 4028 gfp); 4029 if (!new) 4030 return -ENOMEM; 4031 4032 for_each_online_cpu(i) { 4033 new->new[i] = alloc_arraycache(cpu_to_mem(i), limit, 4034 batchcount, gfp); 4035 if (!new->new[i]) { 4036 for (i--; i >= 0; i--) 4037 kfree(new->new[i]); 4038 kfree(new); 4039 return -ENOMEM; 4040 } 4041 } 4042 new->cachep = cachep; 4043 4044 on_each_cpu(do_ccupdate_local, (void *)new, 1); 4045 4046 check_irq_on(); 4047 cachep->batchcount = batchcount; 4048 cachep->limit = limit; 4049 cachep->shared = shared; 4050 4051 for_each_online_cpu(i) { 4052 struct array_cache *ccold = new->new[i]; 4053 if (!ccold) 4054 continue; 4055 spin_lock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock); 4056 free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i)); 4057 spin_unlock_irq(&cachep->nodelists[cpu_to_mem(i)]->list_lock); 4058 kfree(ccold); 4059 } 4060 kfree(new); 4061 return alloc_kmemlist(cachep, gfp); 4062 } 4063 4064 /* Called with cache_chain_mutex held always */ 4065 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) 4066 { 4067 int err; 4068 int limit, shared; 4069 4070 /* 4071 * The head array serves three purposes: 4072 * - create a LIFO ordering, i.e. return objects that are cache-warm 4073 * - reduce the number of spinlock operations. 4074 * - reduce the number of linked list operations on the slab and 4075 * bufctl chains: array operations are cheaper. 4076 * The numbers are guessed, we should auto-tune as described by 4077 * Bonwick. 4078 */ 4079 if (cachep->buffer_size > 131072) 4080 limit = 1; 4081 else if (cachep->buffer_size > PAGE_SIZE) 4082 limit = 8; 4083 else if (cachep->buffer_size > 1024) 4084 limit = 24; 4085 else if (cachep->buffer_size > 256) 4086 limit = 54; 4087 else 4088 limit = 120; 4089 4090 /* 4091 * CPU bound tasks (e.g. network routing) can exhibit cpu bound 4092 * allocation behaviour: Most allocs on one cpu, most free operations 4093 * on another cpu. For these cases, an efficient object passing between 4094 * cpus is necessary. This is provided by a shared array. The array 4095 * replaces Bonwick's magazine layer. 4096 * On uniprocessor, it's functionally equivalent (but less efficient) 4097 * to a larger limit. Thus disabled by default. 4098 */ 4099 shared = 0; 4100 if (cachep->buffer_size <= PAGE_SIZE && num_possible_cpus() > 1) 4101 shared = 8; 4102 4103 #if DEBUG 4104 /* 4105 * With debugging enabled, large batchcount lead to excessively long 4106 * periods with disabled local interrupts. Limit the batchcount 4107 */ 4108 if (limit > 32) 4109 limit = 32; 4110 #endif 4111 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared, gfp); 4112 if (err) 4113 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", 4114 cachep->name, -err); 4115 return err; 4116 } 4117 4118 /* 4119 * Drain an array if it contains any elements taking the l3 lock only if 4120 * necessary. Note that the l3 listlock also protects the array_cache 4121 * if drain_array() is used on the shared array. 4122 */ 4123 static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3, 4124 struct array_cache *ac, int force, int node) 4125 { 4126 int tofree; 4127 4128 if (!ac || !ac->avail) 4129 return; 4130 if (ac->touched && !force) { 4131 ac->touched = 0; 4132 } else { 4133 spin_lock_irq(&l3->list_lock); 4134 if (ac->avail) { 4135 tofree = force ? ac->avail : (ac->limit + 4) / 5; 4136 if (tofree > ac->avail) 4137 tofree = (ac->avail + 1) / 2; 4138 free_block(cachep, ac->entry, tofree, node); 4139 ac->avail -= tofree; 4140 memmove(ac->entry, &(ac->entry[tofree]), 4141 sizeof(void *) * ac->avail); 4142 } 4143 spin_unlock_irq(&l3->list_lock); 4144 } 4145 } 4146 4147 /** 4148 * cache_reap - Reclaim memory from caches. 4149 * @w: work descriptor 4150 * 4151 * Called from workqueue/eventd every few seconds. 4152 * Purpose: 4153 * - clear the per-cpu caches for this CPU. 4154 * - return freeable pages to the main free memory pool. 4155 * 4156 * If we cannot acquire the cache chain mutex then just give up - we'll try 4157 * again on the next iteration. 4158 */ 4159 static void cache_reap(struct work_struct *w) 4160 { 4161 struct kmem_cache *searchp; 4162 struct kmem_list3 *l3; 4163 int node = numa_mem_id(); 4164 struct delayed_work *work = to_delayed_work(w); 4165 4166 if (!mutex_trylock(&cache_chain_mutex)) 4167 /* Give up. Setup the next iteration. */ 4168 goto out; 4169 4170 list_for_each_entry(searchp, &cache_chain, next) { 4171 check_irq_on(); 4172 4173 /* 4174 * We only take the l3 lock if absolutely necessary and we 4175 * have established with reasonable certainty that 4176 * we can do some work if the lock was obtained. 4177 */ 4178 l3 = searchp->nodelists[node]; 4179 4180 reap_alien(searchp, l3); 4181 4182 drain_array(searchp, l3, cpu_cache_get(searchp), 0, node); 4183 4184 /* 4185 * These are racy checks but it does not matter 4186 * if we skip one check or scan twice. 4187 */ 4188 if (time_after(l3->next_reap, jiffies)) 4189 goto next; 4190 4191 l3->next_reap = jiffies + REAPTIMEOUT_LIST3; 4192 4193 drain_array(searchp, l3, l3->shared, 0, node); 4194 4195 if (l3->free_touched) 4196 l3->free_touched = 0; 4197 else { 4198 int freed; 4199 4200 freed = drain_freelist(searchp, l3, (l3->free_limit + 4201 5 * searchp->num - 1) / (5 * searchp->num)); 4202 STATS_ADD_REAPED(searchp, freed); 4203 } 4204 next: 4205 cond_resched(); 4206 } 4207 check_irq_on(); 4208 mutex_unlock(&cache_chain_mutex); 4209 next_reap_node(); 4210 out: 4211 /* Set up the next iteration */ 4212 schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_CPUC)); 4213 } 4214 4215 #ifdef CONFIG_SLABINFO 4216 4217 static void print_slabinfo_header(struct seq_file *m) 4218 { 4219 /* 4220 * Output format version, so at least we can change it 4221 * without _too_ many complaints. 4222 */ 4223 #if STATS 4224 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 4225 #else 4226 seq_puts(m, "slabinfo - version: 2.1\n"); 4227 #endif 4228 seq_puts(m, "# name <active_objs> <num_objs> <objsize> " 4229 "<objperslab> <pagesperslab>"); 4230 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 4231 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 4232 #if STATS 4233 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> " 4234 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 4235 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 4236 #endif 4237 seq_putc(m, '\n'); 4238 } 4239 4240 static void *s_start(struct seq_file *m, loff_t *pos) 4241 { 4242 loff_t n = *pos; 4243 4244 mutex_lock(&cache_chain_mutex); 4245 if (!n) 4246 print_slabinfo_header(m); 4247 4248 return seq_list_start(&cache_chain, *pos); 4249 } 4250 4251 static void *s_next(struct seq_file *m, void *p, loff_t *pos) 4252 { 4253 return seq_list_next(p, &cache_chain, pos); 4254 } 4255 4256 static void s_stop(struct seq_file *m, void *p) 4257 { 4258 mutex_unlock(&cache_chain_mutex); 4259 } 4260 4261 static int s_show(struct seq_file *m, void *p) 4262 { 4263 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); 4264 struct slab *slabp; 4265 unsigned long active_objs; 4266 unsigned long num_objs; 4267 unsigned long active_slabs = 0; 4268 unsigned long num_slabs, free_objects = 0, shared_avail = 0; 4269 const char *name; 4270 char *error = NULL; 4271 int node; 4272 struct kmem_list3 *l3; 4273 4274 active_objs = 0; 4275 num_slabs = 0; 4276 for_each_online_node(node) { 4277 l3 = cachep->nodelists[node]; 4278 if (!l3) 4279 continue; 4280 4281 check_irq_on(); 4282 spin_lock_irq(&l3->list_lock); 4283 4284 list_for_each_entry(slabp, &l3->slabs_full, list) { 4285 if (slabp->inuse != cachep->num && !error) 4286 error = "slabs_full accounting error"; 4287 active_objs += cachep->num; 4288 active_slabs++; 4289 } 4290 list_for_each_entry(slabp, &l3->slabs_partial, list) { 4291 if (slabp->inuse == cachep->num && !error) 4292 error = "slabs_partial inuse accounting error"; 4293 if (!slabp->inuse && !error) 4294 error = "slabs_partial/inuse accounting error"; 4295 active_objs += slabp->inuse; 4296 active_slabs++; 4297 } 4298 list_for_each_entry(slabp, &l3->slabs_free, list) { 4299 if (slabp->inuse && !error) 4300 error = "slabs_free/inuse accounting error"; 4301 num_slabs++; 4302 } 4303 free_objects += l3->free_objects; 4304 if (l3->shared) 4305 shared_avail += l3->shared->avail; 4306 4307 spin_unlock_irq(&l3->list_lock); 4308 } 4309 num_slabs += active_slabs; 4310 num_objs = num_slabs * cachep->num; 4311 if (num_objs - active_objs != free_objects && !error) 4312 error = "free_objects accounting error"; 4313 4314 name = cachep->name; 4315 if (error) 4316 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 4317 4318 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 4319 name, active_objs, num_objs, cachep->buffer_size, 4320 cachep->num, (1 << cachep->gfporder)); 4321 seq_printf(m, " : tunables %4u %4u %4u", 4322 cachep->limit, cachep->batchcount, cachep->shared); 4323 seq_printf(m, " : slabdata %6lu %6lu %6lu", 4324 active_slabs, num_slabs, shared_avail); 4325 #if STATS 4326 { /* list3 stats */ 4327 unsigned long high = cachep->high_mark; 4328 unsigned long allocs = cachep->num_allocations; 4329 unsigned long grown = cachep->grown; 4330 unsigned long reaped = cachep->reaped; 4331 unsigned long errors = cachep->errors; 4332 unsigned long max_freeable = cachep->max_freeable; 4333 unsigned long node_allocs = cachep->node_allocs; 4334 unsigned long node_frees = cachep->node_frees; 4335 unsigned long overflows = cachep->node_overflow; 4336 4337 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu " 4338 "%4lu %4lu %4lu %4lu %4lu", 4339 allocs, high, grown, 4340 reaped, errors, max_freeable, node_allocs, 4341 node_frees, overflows); 4342 } 4343 /* cpu stats */ 4344 { 4345 unsigned long allochit = atomic_read(&cachep->allochit); 4346 unsigned long allocmiss = atomic_read(&cachep->allocmiss); 4347 unsigned long freehit = atomic_read(&cachep->freehit); 4348 unsigned long freemiss = atomic_read(&cachep->freemiss); 4349 4350 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", 4351 allochit, allocmiss, freehit, freemiss); 4352 } 4353 #endif 4354 seq_putc(m, '\n'); 4355 return 0; 4356 } 4357 4358 /* 4359 * slabinfo_op - iterator that generates /proc/slabinfo 4360 * 4361 * Output layout: 4362 * cache-name 4363 * num-active-objs 4364 * total-objs 4365 * object size 4366 * num-active-slabs 4367 * total-slabs 4368 * num-pages-per-slab 4369 * + further values on SMP and with statistics enabled 4370 */ 4371 4372 static const struct seq_operations slabinfo_op = { 4373 .start = s_start, 4374 .next = s_next, 4375 .stop = s_stop, 4376 .show = s_show, 4377 }; 4378 4379 #define MAX_SLABINFO_WRITE 128 4380 /** 4381 * slabinfo_write - Tuning for the slab allocator 4382 * @file: unused 4383 * @buffer: user buffer 4384 * @count: data length 4385 * @ppos: unused 4386 */ 4387 static ssize_t slabinfo_write(struct file *file, const char __user *buffer, 4388 size_t count, loff_t *ppos) 4389 { 4390 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; 4391 int limit, batchcount, shared, res; 4392 struct kmem_cache *cachep; 4393 4394 if (count > MAX_SLABINFO_WRITE) 4395 return -EINVAL; 4396 if (copy_from_user(&kbuf, buffer, count)) 4397 return -EFAULT; 4398 kbuf[MAX_SLABINFO_WRITE] = '\0'; 4399 4400 tmp = strchr(kbuf, ' '); 4401 if (!tmp) 4402 return -EINVAL; 4403 *tmp = '\0'; 4404 tmp++; 4405 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3) 4406 return -EINVAL; 4407 4408 /* Find the cache in the chain of caches. */ 4409 mutex_lock(&cache_chain_mutex); 4410 res = -EINVAL; 4411 list_for_each_entry(cachep, &cache_chain, next) { 4412 if (!strcmp(cachep->name, kbuf)) { 4413 if (limit < 1 || batchcount < 1 || 4414 batchcount > limit || shared < 0) { 4415 res = 0; 4416 } else { 4417 res = do_tune_cpucache(cachep, limit, 4418 batchcount, shared, 4419 GFP_KERNEL); 4420 } 4421 break; 4422 } 4423 } 4424 mutex_unlock(&cache_chain_mutex); 4425 if (res >= 0) 4426 res = count; 4427 return res; 4428 } 4429 4430 static int slabinfo_open(struct inode *inode, struct file *file) 4431 { 4432 return seq_open(file, &slabinfo_op); 4433 } 4434 4435 static const struct file_operations proc_slabinfo_operations = { 4436 .open = slabinfo_open, 4437 .read = seq_read, 4438 .write = slabinfo_write, 4439 .llseek = seq_lseek, 4440 .release = seq_release, 4441 }; 4442 4443 #ifdef CONFIG_DEBUG_SLAB_LEAK 4444 4445 static void *leaks_start(struct seq_file *m, loff_t *pos) 4446 { 4447 mutex_lock(&cache_chain_mutex); 4448 return seq_list_start(&cache_chain, *pos); 4449 } 4450 4451 static inline int add_caller(unsigned long *n, unsigned long v) 4452 { 4453 unsigned long *p; 4454 int l; 4455 if (!v) 4456 return 1; 4457 l = n[1]; 4458 p = n + 2; 4459 while (l) { 4460 int i = l/2; 4461 unsigned long *q = p + 2 * i; 4462 if (*q == v) { 4463 q[1]++; 4464 return 1; 4465 } 4466 if (*q > v) { 4467 l = i; 4468 } else { 4469 p = q + 2; 4470 l -= i + 1; 4471 } 4472 } 4473 if (++n[1] == n[0]) 4474 return 0; 4475 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n)); 4476 p[0] = v; 4477 p[1] = 1; 4478 return 1; 4479 } 4480 4481 static void handle_slab(unsigned long *n, struct kmem_cache *c, struct slab *s) 4482 { 4483 void *p; 4484 int i; 4485 if (n[0] == n[1]) 4486 return; 4487 for (i = 0, p = s->s_mem; i < c->num; i++, p += c->buffer_size) { 4488 if (slab_bufctl(s)[i] != BUFCTL_ACTIVE) 4489 continue; 4490 if (!add_caller(n, (unsigned long)*dbg_userword(c, p))) 4491 return; 4492 } 4493 } 4494 4495 static void show_symbol(struct seq_file *m, unsigned long address) 4496 { 4497 #ifdef CONFIG_KALLSYMS 4498 unsigned long offset, size; 4499 char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN]; 4500 4501 if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) { 4502 seq_printf(m, "%s+%#lx/%#lx", name, offset, size); 4503 if (modname[0]) 4504 seq_printf(m, " [%s]", modname); 4505 return; 4506 } 4507 #endif 4508 seq_printf(m, "%p", (void *)address); 4509 } 4510 4511 static int leaks_show(struct seq_file *m, void *p) 4512 { 4513 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, next); 4514 struct slab *slabp; 4515 struct kmem_list3 *l3; 4516 const char *name; 4517 unsigned long *n = m->private; 4518 int node; 4519 int i; 4520 4521 if (!(cachep->flags & SLAB_STORE_USER)) 4522 return 0; 4523 if (!(cachep->flags & SLAB_RED_ZONE)) 4524 return 0; 4525 4526 /* OK, we can do it */ 4527 4528 n[1] = 0; 4529 4530 for_each_online_node(node) { 4531 l3 = cachep->nodelists[node]; 4532 if (!l3) 4533 continue; 4534 4535 check_irq_on(); 4536 spin_lock_irq(&l3->list_lock); 4537 4538 list_for_each_entry(slabp, &l3->slabs_full, list) 4539 handle_slab(n, cachep, slabp); 4540 list_for_each_entry(slabp, &l3->slabs_partial, list) 4541 handle_slab(n, cachep, slabp); 4542 spin_unlock_irq(&l3->list_lock); 4543 } 4544 name = cachep->name; 4545 if (n[0] == n[1]) { 4546 /* Increase the buffer size */ 4547 mutex_unlock(&cache_chain_mutex); 4548 m->private = kzalloc(n[0] * 4 * sizeof(unsigned long), GFP_KERNEL); 4549 if (!m->private) { 4550 /* Too bad, we are really out */ 4551 m->private = n; 4552 mutex_lock(&cache_chain_mutex); 4553 return -ENOMEM; 4554 } 4555 *(unsigned long *)m->private = n[0] * 2; 4556 kfree(n); 4557 mutex_lock(&cache_chain_mutex); 4558 /* Now make sure this entry will be retried */ 4559 m->count = m->size; 4560 return 0; 4561 } 4562 for (i = 0; i < n[1]; i++) { 4563 seq_printf(m, "%s: %lu ", name, n[2*i+3]); 4564 show_symbol(m, n[2*i+2]); 4565 seq_putc(m, '\n'); 4566 } 4567 4568 return 0; 4569 } 4570 4571 static const struct seq_operations slabstats_op = { 4572 .start = leaks_start, 4573 .next = s_next, 4574 .stop = s_stop, 4575 .show = leaks_show, 4576 }; 4577 4578 static int slabstats_open(struct inode *inode, struct file *file) 4579 { 4580 unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL); 4581 int ret = -ENOMEM; 4582 if (n) { 4583 ret = seq_open(file, &slabstats_op); 4584 if (!ret) { 4585 struct seq_file *m = file->private_data; 4586 *n = PAGE_SIZE / (2 * sizeof(unsigned long)); 4587 m->private = n; 4588 n = NULL; 4589 } 4590 kfree(n); 4591 } 4592 return ret; 4593 } 4594 4595 static const struct file_operations proc_slabstats_operations = { 4596 .open = slabstats_open, 4597 .read = seq_read, 4598 .llseek = seq_lseek, 4599 .release = seq_release_private, 4600 }; 4601 #endif 4602 4603 static int __init slab_proc_init(void) 4604 { 4605 proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations); 4606 #ifdef CONFIG_DEBUG_SLAB_LEAK 4607 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations); 4608 #endif 4609 return 0; 4610 } 4611 module_init(slab_proc_init); 4612 #endif 4613 4614 /** 4615 * ksize - get the actual amount of memory allocated for a given object 4616 * @objp: Pointer to the object 4617 * 4618 * kmalloc may internally round up allocations and return more memory 4619 * than requested. ksize() can be used to determine the actual amount of 4620 * memory allocated. The caller may use this additional memory, even though 4621 * a smaller amount of memory was initially specified with the kmalloc call. 4622 * The caller must guarantee that objp points to a valid object previously 4623 * allocated with either kmalloc() or kmem_cache_alloc(). The object 4624 * must not be freed during the duration of the call. 4625 */ 4626 size_t ksize(const void *objp) 4627 { 4628 BUG_ON(!objp); 4629 if (unlikely(objp == ZERO_SIZE_PTR)) 4630 return 0; 4631 4632 return obj_size(virt_to_cache(objp)); 4633 } 4634 EXPORT_SYMBOL(ksize); 4635