1 /* 2 * linux/mm/slab.c 3 * Written by Mark Hemment, 1996/97. 4 * (markhe@nextd.demon.co.uk) 5 * 6 * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli 7 * 8 * Major cleanup, different bufctl logic, per-cpu arrays 9 * (c) 2000 Manfred Spraul 10 * 11 * Cleanup, make the head arrays unconditional, preparation for NUMA 12 * (c) 2002 Manfred Spraul 13 * 14 * An implementation of the Slab Allocator as described in outline in; 15 * UNIX Internals: The New Frontiers by Uresh Vahalia 16 * Pub: Prentice Hall ISBN 0-13-101908-2 17 * or with a little more detail in; 18 * The Slab Allocator: An Object-Caching Kernel Memory Allocator 19 * Jeff Bonwick (Sun Microsystems). 20 * Presented at: USENIX Summer 1994 Technical Conference 21 * 22 * The memory is organized in caches, one cache for each object type. 23 * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct) 24 * Each cache consists out of many slabs (they are small (usually one 25 * page long) and always contiguous), and each slab contains multiple 26 * initialized objects. 27 * 28 * This means, that your constructor is used only for newly allocated 29 * slabs and you must pass objects with the same initializations to 30 * kmem_cache_free. 31 * 32 * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM, 33 * normal). If you need a special memory type, then must create a new 34 * cache for that memory type. 35 * 36 * In order to reduce fragmentation, the slabs are sorted in 3 groups: 37 * full slabs with 0 free objects 38 * partial slabs 39 * empty slabs with no allocated objects 40 * 41 * If partial slabs exist, then new allocations come from these slabs, 42 * otherwise from empty slabs or new slabs are allocated. 43 * 44 * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache 45 * during kmem_cache_destroy(). The caller must prevent concurrent allocs. 46 * 47 * Each cache has a short per-cpu head array, most allocs 48 * and frees go into that array, and if that array overflows, then 1/2 49 * of the entries in the array are given back into the global cache. 50 * The head array is strictly LIFO and should improve the cache hit rates. 51 * On SMP, it additionally reduces the spinlock operations. 52 * 53 * The c_cpuarray may not be read with enabled local interrupts - 54 * it's changed with a smp_call_function(). 55 * 56 * SMP synchronization: 57 * constructors and destructors are called without any locking. 58 * Several members in struct kmem_cache and struct slab never change, they 59 * are accessed without any locking. 60 * The per-cpu arrays are never accessed from the wrong cpu, no locking, 61 * and local interrupts are disabled so slab code is preempt-safe. 62 * The non-constant members are protected with a per-cache irq spinlock. 63 * 64 * Many thanks to Mark Hemment, who wrote another per-cpu slab patch 65 * in 2000 - many ideas in the current implementation are derived from 66 * his patch. 67 * 68 * Further notes from the original documentation: 69 * 70 * 11 April '97. Started multi-threading - markhe 71 * The global cache-chain is protected by the mutex 'slab_mutex'. 72 * The sem is only needed when accessing/extending the cache-chain, which 73 * can never happen inside an interrupt (kmem_cache_create(), 74 * kmem_cache_shrink() and kmem_cache_reap()). 75 * 76 * At present, each engine can be growing a cache. This should be blocked. 77 * 78 * 15 March 2005. NUMA slab allocator. 79 * Shai Fultheim <shai@scalex86.org>. 80 * Shobhit Dayal <shobhit@calsoftinc.com> 81 * Alok N Kataria <alokk@calsoftinc.com> 82 * Christoph Lameter <christoph@lameter.com> 83 * 84 * Modified the slab allocator to be node aware on NUMA systems. 85 * Each node has its own list of partial, free and full slabs. 86 * All object allocations for a node occur from node specific slab lists. 87 */ 88 89 #include <linux/slab.h> 90 #include <linux/mm.h> 91 #include <linux/poison.h> 92 #include <linux/swap.h> 93 #include <linux/cache.h> 94 #include <linux/interrupt.h> 95 #include <linux/init.h> 96 #include <linux/compiler.h> 97 #include <linux/cpuset.h> 98 #include <linux/proc_fs.h> 99 #include <linux/seq_file.h> 100 #include <linux/notifier.h> 101 #include <linux/kallsyms.h> 102 #include <linux/cpu.h> 103 #include <linux/sysctl.h> 104 #include <linux/module.h> 105 #include <linux/rcupdate.h> 106 #include <linux/string.h> 107 #include <linux/uaccess.h> 108 #include <linux/nodemask.h> 109 #include <linux/kmemleak.h> 110 #include <linux/mempolicy.h> 111 #include <linux/mutex.h> 112 #include <linux/fault-inject.h> 113 #include <linux/rtmutex.h> 114 #include <linux/reciprocal_div.h> 115 #include <linux/debugobjects.h> 116 #include <linux/kmemcheck.h> 117 #include <linux/memory.h> 118 #include <linux/prefetch.h> 119 120 #include <net/sock.h> 121 122 #include <asm/cacheflush.h> 123 #include <asm/tlbflush.h> 124 #include <asm/page.h> 125 126 #include <trace/events/kmem.h> 127 128 #include "internal.h" 129 130 #include "slab.h" 131 132 /* 133 * DEBUG - 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON. 134 * 0 for faster, smaller code (especially in the critical paths). 135 * 136 * STATS - 1 to collect stats for /proc/slabinfo. 137 * 0 for faster, smaller code (especially in the critical paths). 138 * 139 * FORCED_DEBUG - 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible) 140 */ 141 142 #ifdef CONFIG_DEBUG_SLAB 143 #define DEBUG 1 144 #define STATS 1 145 #define FORCED_DEBUG 1 146 #else 147 #define DEBUG 0 148 #define STATS 0 149 #define FORCED_DEBUG 0 150 #endif 151 152 /* Shouldn't this be in a header file somewhere? */ 153 #define BYTES_PER_WORD sizeof(void *) 154 #define REDZONE_ALIGN max(BYTES_PER_WORD, __alignof__(unsigned long long)) 155 156 #ifndef ARCH_KMALLOC_FLAGS 157 #define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN 158 #endif 159 160 #define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \ 161 <= SLAB_OBJ_MIN_SIZE) ? 1 : 0) 162 163 #if FREELIST_BYTE_INDEX 164 typedef unsigned char freelist_idx_t; 165 #else 166 typedef unsigned short freelist_idx_t; 167 #endif 168 169 #define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1) 170 171 /* 172 * true if a page was allocated from pfmemalloc reserves for network-based 173 * swap 174 */ 175 static bool pfmemalloc_active __read_mostly; 176 177 /* 178 * struct array_cache 179 * 180 * Purpose: 181 * - LIFO ordering, to hand out cache-warm objects from _alloc 182 * - reduce the number of linked list operations 183 * - reduce spinlock operations 184 * 185 * The limit is stored in the per-cpu structure to reduce the data cache 186 * footprint. 187 * 188 */ 189 struct array_cache { 190 unsigned int avail; 191 unsigned int limit; 192 unsigned int batchcount; 193 unsigned int touched; 194 spinlock_t lock; 195 void *entry[]; /* 196 * Must have this definition in here for the proper 197 * alignment of array_cache. Also simplifies accessing 198 * the entries. 199 * 200 * Entries should not be directly dereferenced as 201 * entries belonging to slabs marked pfmemalloc will 202 * have the lower bits set SLAB_OBJ_PFMEMALLOC 203 */ 204 }; 205 206 #define SLAB_OBJ_PFMEMALLOC 1 207 static inline bool is_obj_pfmemalloc(void *objp) 208 { 209 return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC; 210 } 211 212 static inline void set_obj_pfmemalloc(void **objp) 213 { 214 *objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC); 215 return; 216 } 217 218 static inline void clear_obj_pfmemalloc(void **objp) 219 { 220 *objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC); 221 } 222 223 /* 224 * bootstrap: The caches do not work without cpuarrays anymore, but the 225 * cpuarrays are allocated from the generic caches... 226 */ 227 #define BOOT_CPUCACHE_ENTRIES 1 228 struct arraycache_init { 229 struct array_cache cache; 230 void *entries[BOOT_CPUCACHE_ENTRIES]; 231 }; 232 233 /* 234 * Need this for bootstrapping a per node allocator. 235 */ 236 #define NUM_INIT_LISTS (3 * MAX_NUMNODES) 237 static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS]; 238 #define CACHE_CACHE 0 239 #define SIZE_AC MAX_NUMNODES 240 #define SIZE_NODE (2 * MAX_NUMNODES) 241 242 static int drain_freelist(struct kmem_cache *cache, 243 struct kmem_cache_node *n, int tofree); 244 static void free_block(struct kmem_cache *cachep, void **objpp, int len, 245 int node); 246 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp); 247 static void cache_reap(struct work_struct *unused); 248 249 static int slab_early_init = 1; 250 251 #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init)) 252 #define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node)) 253 254 static void kmem_cache_node_init(struct kmem_cache_node *parent) 255 { 256 INIT_LIST_HEAD(&parent->slabs_full); 257 INIT_LIST_HEAD(&parent->slabs_partial); 258 INIT_LIST_HEAD(&parent->slabs_free); 259 parent->shared = NULL; 260 parent->alien = NULL; 261 parent->colour_next = 0; 262 spin_lock_init(&parent->list_lock); 263 parent->free_objects = 0; 264 parent->free_touched = 0; 265 } 266 267 #define MAKE_LIST(cachep, listp, slab, nodeid) \ 268 do { \ 269 INIT_LIST_HEAD(listp); \ 270 list_splice(&(cachep->node[nodeid]->slab), listp); \ 271 } while (0) 272 273 #define MAKE_ALL_LISTS(cachep, ptr, nodeid) \ 274 do { \ 275 MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid); \ 276 MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \ 277 MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid); \ 278 } while (0) 279 280 #define CFLGS_OFF_SLAB (0x80000000UL) 281 #define OFF_SLAB(x) ((x)->flags & CFLGS_OFF_SLAB) 282 283 #define BATCHREFILL_LIMIT 16 284 /* 285 * Optimization question: fewer reaps means less probability for unnessary 286 * cpucache drain/refill cycles. 287 * 288 * OTOH the cpuarrays can contain lots of objects, 289 * which could lock up otherwise freeable slabs. 290 */ 291 #define REAPTIMEOUT_AC (2*HZ) 292 #define REAPTIMEOUT_NODE (4*HZ) 293 294 #if STATS 295 #define STATS_INC_ACTIVE(x) ((x)->num_active++) 296 #define STATS_DEC_ACTIVE(x) ((x)->num_active--) 297 #define STATS_INC_ALLOCED(x) ((x)->num_allocations++) 298 #define STATS_INC_GROWN(x) ((x)->grown++) 299 #define STATS_ADD_REAPED(x,y) ((x)->reaped += (y)) 300 #define STATS_SET_HIGH(x) \ 301 do { \ 302 if ((x)->num_active > (x)->high_mark) \ 303 (x)->high_mark = (x)->num_active; \ 304 } while (0) 305 #define STATS_INC_ERR(x) ((x)->errors++) 306 #define STATS_INC_NODEALLOCS(x) ((x)->node_allocs++) 307 #define STATS_INC_NODEFREES(x) ((x)->node_frees++) 308 #define STATS_INC_ACOVERFLOW(x) ((x)->node_overflow++) 309 #define STATS_SET_FREEABLE(x, i) \ 310 do { \ 311 if ((x)->max_freeable < i) \ 312 (x)->max_freeable = i; \ 313 } while (0) 314 #define STATS_INC_ALLOCHIT(x) atomic_inc(&(x)->allochit) 315 #define STATS_INC_ALLOCMISS(x) atomic_inc(&(x)->allocmiss) 316 #define STATS_INC_FREEHIT(x) atomic_inc(&(x)->freehit) 317 #define STATS_INC_FREEMISS(x) atomic_inc(&(x)->freemiss) 318 #else 319 #define STATS_INC_ACTIVE(x) do { } while (0) 320 #define STATS_DEC_ACTIVE(x) do { } while (0) 321 #define STATS_INC_ALLOCED(x) do { } while (0) 322 #define STATS_INC_GROWN(x) do { } while (0) 323 #define STATS_ADD_REAPED(x,y) do { (void)(y); } while (0) 324 #define STATS_SET_HIGH(x) do { } while (0) 325 #define STATS_INC_ERR(x) do { } while (0) 326 #define STATS_INC_NODEALLOCS(x) do { } while (0) 327 #define STATS_INC_NODEFREES(x) do { } while (0) 328 #define STATS_INC_ACOVERFLOW(x) do { } while (0) 329 #define STATS_SET_FREEABLE(x, i) do { } while (0) 330 #define STATS_INC_ALLOCHIT(x) do { } while (0) 331 #define STATS_INC_ALLOCMISS(x) do { } while (0) 332 #define STATS_INC_FREEHIT(x) do { } while (0) 333 #define STATS_INC_FREEMISS(x) do { } while (0) 334 #endif 335 336 #if DEBUG 337 338 /* 339 * memory layout of objects: 340 * 0 : objp 341 * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that 342 * the end of an object is aligned with the end of the real 343 * allocation. Catches writes behind the end of the allocation. 344 * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1: 345 * redzone word. 346 * cachep->obj_offset: The real object. 347 * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long] 348 * cachep->size - 1* BYTES_PER_WORD: last caller address 349 * [BYTES_PER_WORD long] 350 */ 351 static int obj_offset(struct kmem_cache *cachep) 352 { 353 return cachep->obj_offset; 354 } 355 356 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp) 357 { 358 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 359 return (unsigned long long*) (objp + obj_offset(cachep) - 360 sizeof(unsigned long long)); 361 } 362 363 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp) 364 { 365 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 366 if (cachep->flags & SLAB_STORE_USER) 367 return (unsigned long long *)(objp + cachep->size - 368 sizeof(unsigned long long) - 369 REDZONE_ALIGN); 370 return (unsigned long long *) (objp + cachep->size - 371 sizeof(unsigned long long)); 372 } 373 374 static void **dbg_userword(struct kmem_cache *cachep, void *objp) 375 { 376 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); 377 return (void **)(objp + cachep->size - BYTES_PER_WORD); 378 } 379 380 #else 381 382 #define obj_offset(x) 0 383 #define dbg_redzone1(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) 384 #define dbg_redzone2(cachep, objp) ({BUG(); (unsigned long long *)NULL;}) 385 #define dbg_userword(cachep, objp) ({BUG(); (void **)NULL;}) 386 387 #endif 388 389 /* 390 * Do not go above this order unless 0 objects fit into the slab or 391 * overridden on the command line. 392 */ 393 #define SLAB_MAX_ORDER_HI 1 394 #define SLAB_MAX_ORDER_LO 0 395 static int slab_max_order = SLAB_MAX_ORDER_LO; 396 static bool slab_max_order_set __initdata; 397 398 static inline struct kmem_cache *virt_to_cache(const void *obj) 399 { 400 struct page *page = virt_to_head_page(obj); 401 return page->slab_cache; 402 } 403 404 static inline void *index_to_obj(struct kmem_cache *cache, struct page *page, 405 unsigned int idx) 406 { 407 return page->s_mem + cache->size * idx; 408 } 409 410 /* 411 * We want to avoid an expensive divide : (offset / cache->size) 412 * Using the fact that size is a constant for a particular cache, 413 * we can replace (offset / cache->size) by 414 * reciprocal_divide(offset, cache->reciprocal_buffer_size) 415 */ 416 static inline unsigned int obj_to_index(const struct kmem_cache *cache, 417 const struct page *page, void *obj) 418 { 419 u32 offset = (obj - page->s_mem); 420 return reciprocal_divide(offset, cache->reciprocal_buffer_size); 421 } 422 423 static struct arraycache_init initarray_generic = 424 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 425 426 /* internal cache of cache description objs */ 427 static struct kmem_cache kmem_cache_boot = { 428 .batchcount = 1, 429 .limit = BOOT_CPUCACHE_ENTRIES, 430 .shared = 1, 431 .size = sizeof(struct kmem_cache), 432 .name = "kmem_cache", 433 }; 434 435 #define BAD_ALIEN_MAGIC 0x01020304ul 436 437 #ifdef CONFIG_LOCKDEP 438 439 /* 440 * Slab sometimes uses the kmalloc slabs to store the slab headers 441 * for other slabs "off slab". 442 * The locking for this is tricky in that it nests within the locks 443 * of all other slabs in a few places; to deal with this special 444 * locking we put on-slab caches into a separate lock-class. 445 * 446 * We set lock class for alien array caches which are up during init. 447 * The lock annotation will be lost if all cpus of a node goes down and 448 * then comes back up during hotplug 449 */ 450 static struct lock_class_key on_slab_l3_key; 451 static struct lock_class_key on_slab_alc_key; 452 453 static struct lock_class_key debugobj_l3_key; 454 static struct lock_class_key debugobj_alc_key; 455 456 static void slab_set_lock_classes(struct kmem_cache *cachep, 457 struct lock_class_key *l3_key, struct lock_class_key *alc_key, 458 int q) 459 { 460 struct array_cache **alc; 461 struct kmem_cache_node *n; 462 int r; 463 464 n = cachep->node[q]; 465 if (!n) 466 return; 467 468 lockdep_set_class(&n->list_lock, l3_key); 469 alc = n->alien; 470 /* 471 * FIXME: This check for BAD_ALIEN_MAGIC 472 * should go away when common slab code is taught to 473 * work even without alien caches. 474 * Currently, non NUMA code returns BAD_ALIEN_MAGIC 475 * for alloc_alien_cache, 476 */ 477 if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC) 478 return; 479 for_each_node(r) { 480 if (alc[r]) 481 lockdep_set_class(&alc[r]->lock, alc_key); 482 } 483 } 484 485 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) 486 { 487 slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node); 488 } 489 490 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) 491 { 492 int node; 493 494 for_each_online_node(node) 495 slab_set_debugobj_lock_classes_node(cachep, node); 496 } 497 498 static void init_node_lock_keys(int q) 499 { 500 int i; 501 502 if (slab_state < UP) 503 return; 504 505 for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) { 506 struct kmem_cache_node *n; 507 struct kmem_cache *cache = kmalloc_caches[i]; 508 509 if (!cache) 510 continue; 511 512 n = cache->node[q]; 513 if (!n || OFF_SLAB(cache)) 514 continue; 515 516 slab_set_lock_classes(cache, &on_slab_l3_key, 517 &on_slab_alc_key, q); 518 } 519 } 520 521 static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q) 522 { 523 if (!cachep->node[q]) 524 return; 525 526 slab_set_lock_classes(cachep, &on_slab_l3_key, 527 &on_slab_alc_key, q); 528 } 529 530 static inline void on_slab_lock_classes(struct kmem_cache *cachep) 531 { 532 int node; 533 534 VM_BUG_ON(OFF_SLAB(cachep)); 535 for_each_node(node) 536 on_slab_lock_classes_node(cachep, node); 537 } 538 539 static inline void init_lock_keys(void) 540 { 541 int node; 542 543 for_each_node(node) 544 init_node_lock_keys(node); 545 } 546 #else 547 static void init_node_lock_keys(int q) 548 { 549 } 550 551 static inline void init_lock_keys(void) 552 { 553 } 554 555 static inline void on_slab_lock_classes(struct kmem_cache *cachep) 556 { 557 } 558 559 static inline void on_slab_lock_classes_node(struct kmem_cache *cachep, int node) 560 { 561 } 562 563 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node) 564 { 565 } 566 567 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep) 568 { 569 } 570 #endif 571 572 static DEFINE_PER_CPU(struct delayed_work, slab_reap_work); 573 574 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep) 575 { 576 return cachep->array[smp_processor_id()]; 577 } 578 579 static int calculate_nr_objs(size_t slab_size, size_t buffer_size, 580 size_t idx_size, size_t align) 581 { 582 int nr_objs; 583 size_t freelist_size; 584 585 /* 586 * Ignore padding for the initial guess. The padding 587 * is at most @align-1 bytes, and @buffer_size is at 588 * least @align. In the worst case, this result will 589 * be one greater than the number of objects that fit 590 * into the memory allocation when taking the padding 591 * into account. 592 */ 593 nr_objs = slab_size / (buffer_size + idx_size); 594 595 /* 596 * This calculated number will be either the right 597 * amount, or one greater than what we want. 598 */ 599 freelist_size = slab_size - nr_objs * buffer_size; 600 if (freelist_size < ALIGN(nr_objs * idx_size, align)) 601 nr_objs--; 602 603 return nr_objs; 604 } 605 606 /* 607 * Calculate the number of objects and left-over bytes for a given buffer size. 608 */ 609 static void cache_estimate(unsigned long gfporder, size_t buffer_size, 610 size_t align, int flags, size_t *left_over, 611 unsigned int *num) 612 { 613 int nr_objs; 614 size_t mgmt_size; 615 size_t slab_size = PAGE_SIZE << gfporder; 616 617 /* 618 * The slab management structure can be either off the slab or 619 * on it. For the latter case, the memory allocated for a 620 * slab is used for: 621 * 622 * - One unsigned int for each object 623 * - Padding to respect alignment of @align 624 * - @buffer_size bytes for each object 625 * 626 * If the slab management structure is off the slab, then the 627 * alignment will already be calculated into the size. Because 628 * the slabs are all pages aligned, the objects will be at the 629 * correct alignment when allocated. 630 */ 631 if (flags & CFLGS_OFF_SLAB) { 632 mgmt_size = 0; 633 nr_objs = slab_size / buffer_size; 634 635 } else { 636 nr_objs = calculate_nr_objs(slab_size, buffer_size, 637 sizeof(freelist_idx_t), align); 638 mgmt_size = ALIGN(nr_objs * sizeof(freelist_idx_t), align); 639 } 640 *num = nr_objs; 641 *left_over = slab_size - nr_objs*buffer_size - mgmt_size; 642 } 643 644 #if DEBUG 645 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg) 646 647 static void __slab_error(const char *function, struct kmem_cache *cachep, 648 char *msg) 649 { 650 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", 651 function, cachep->name, msg); 652 dump_stack(); 653 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 654 } 655 #endif 656 657 /* 658 * By default on NUMA we use alien caches to stage the freeing of 659 * objects allocated from other nodes. This causes massive memory 660 * inefficiencies when using fake NUMA setup to split memory into a 661 * large number of small nodes, so it can be disabled on the command 662 * line 663 */ 664 665 static int use_alien_caches __read_mostly = 1; 666 static int __init noaliencache_setup(char *s) 667 { 668 use_alien_caches = 0; 669 return 1; 670 } 671 __setup("noaliencache", noaliencache_setup); 672 673 static int __init slab_max_order_setup(char *str) 674 { 675 get_option(&str, &slab_max_order); 676 slab_max_order = slab_max_order < 0 ? 0 : 677 min(slab_max_order, MAX_ORDER - 1); 678 slab_max_order_set = true; 679 680 return 1; 681 } 682 __setup("slab_max_order=", slab_max_order_setup); 683 684 #ifdef CONFIG_NUMA 685 /* 686 * Special reaping functions for NUMA systems called from cache_reap(). 687 * These take care of doing round robin flushing of alien caches (containing 688 * objects freed on different nodes from which they were allocated) and the 689 * flushing of remote pcps by calling drain_node_pages. 690 */ 691 static DEFINE_PER_CPU(unsigned long, slab_reap_node); 692 693 static void init_reap_node(int cpu) 694 { 695 int node; 696 697 node = next_node(cpu_to_mem(cpu), node_online_map); 698 if (node == MAX_NUMNODES) 699 node = first_node(node_online_map); 700 701 per_cpu(slab_reap_node, cpu) = node; 702 } 703 704 static void next_reap_node(void) 705 { 706 int node = __this_cpu_read(slab_reap_node); 707 708 node = next_node(node, node_online_map); 709 if (unlikely(node >= MAX_NUMNODES)) 710 node = first_node(node_online_map); 711 __this_cpu_write(slab_reap_node, node); 712 } 713 714 #else 715 #define init_reap_node(cpu) do { } while (0) 716 #define next_reap_node(void) do { } while (0) 717 #endif 718 719 /* 720 * Initiate the reap timer running on the target CPU. We run at around 1 to 2Hz 721 * via the workqueue/eventd. 722 * Add the CPU number into the expiration time to minimize the possibility of 723 * the CPUs getting into lockstep and contending for the global cache chain 724 * lock. 725 */ 726 static void start_cpu_timer(int cpu) 727 { 728 struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu); 729 730 /* 731 * When this gets called from do_initcalls via cpucache_init(), 732 * init_workqueues() has already run, so keventd will be setup 733 * at that time. 734 */ 735 if (keventd_up() && reap_work->work.func == NULL) { 736 init_reap_node(cpu); 737 INIT_DEFERRABLE_WORK(reap_work, cache_reap); 738 schedule_delayed_work_on(cpu, reap_work, 739 __round_jiffies_relative(HZ, cpu)); 740 } 741 } 742 743 static struct array_cache *alloc_arraycache(int node, int entries, 744 int batchcount, gfp_t gfp) 745 { 746 int memsize = sizeof(void *) * entries + sizeof(struct array_cache); 747 struct array_cache *nc = NULL; 748 749 nc = kmalloc_node(memsize, gfp, node); 750 /* 751 * The array_cache structures contain pointers to free object. 752 * However, when such objects are allocated or transferred to another 753 * cache the pointers are not cleared and they could be counted as 754 * valid references during a kmemleak scan. Therefore, kmemleak must 755 * not scan such objects. 756 */ 757 kmemleak_no_scan(nc); 758 if (nc) { 759 nc->avail = 0; 760 nc->limit = entries; 761 nc->batchcount = batchcount; 762 nc->touched = 0; 763 spin_lock_init(&nc->lock); 764 } 765 return nc; 766 } 767 768 static inline bool is_slab_pfmemalloc(struct page *page) 769 { 770 return PageSlabPfmemalloc(page); 771 } 772 773 /* Clears pfmemalloc_active if no slabs have pfmalloc set */ 774 static void recheck_pfmemalloc_active(struct kmem_cache *cachep, 775 struct array_cache *ac) 776 { 777 struct kmem_cache_node *n = cachep->node[numa_mem_id()]; 778 struct page *page; 779 unsigned long flags; 780 781 if (!pfmemalloc_active) 782 return; 783 784 spin_lock_irqsave(&n->list_lock, flags); 785 list_for_each_entry(page, &n->slabs_full, lru) 786 if (is_slab_pfmemalloc(page)) 787 goto out; 788 789 list_for_each_entry(page, &n->slabs_partial, lru) 790 if (is_slab_pfmemalloc(page)) 791 goto out; 792 793 list_for_each_entry(page, &n->slabs_free, lru) 794 if (is_slab_pfmemalloc(page)) 795 goto out; 796 797 pfmemalloc_active = false; 798 out: 799 spin_unlock_irqrestore(&n->list_lock, flags); 800 } 801 802 static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac, 803 gfp_t flags, bool force_refill) 804 { 805 int i; 806 void *objp = ac->entry[--ac->avail]; 807 808 /* Ensure the caller is allowed to use objects from PFMEMALLOC slab */ 809 if (unlikely(is_obj_pfmemalloc(objp))) { 810 struct kmem_cache_node *n; 811 812 if (gfp_pfmemalloc_allowed(flags)) { 813 clear_obj_pfmemalloc(&objp); 814 return objp; 815 } 816 817 /* The caller cannot use PFMEMALLOC objects, find another one */ 818 for (i = 0; i < ac->avail; i++) { 819 /* If a !PFMEMALLOC object is found, swap them */ 820 if (!is_obj_pfmemalloc(ac->entry[i])) { 821 objp = ac->entry[i]; 822 ac->entry[i] = ac->entry[ac->avail]; 823 ac->entry[ac->avail] = objp; 824 return objp; 825 } 826 } 827 828 /* 829 * If there are empty slabs on the slabs_free list and we are 830 * being forced to refill the cache, mark this one !pfmemalloc. 831 */ 832 n = cachep->node[numa_mem_id()]; 833 if (!list_empty(&n->slabs_free) && force_refill) { 834 struct page *page = virt_to_head_page(objp); 835 ClearPageSlabPfmemalloc(page); 836 clear_obj_pfmemalloc(&objp); 837 recheck_pfmemalloc_active(cachep, ac); 838 return objp; 839 } 840 841 /* No !PFMEMALLOC objects available */ 842 ac->avail++; 843 objp = NULL; 844 } 845 846 return objp; 847 } 848 849 static inline void *ac_get_obj(struct kmem_cache *cachep, 850 struct array_cache *ac, gfp_t flags, bool force_refill) 851 { 852 void *objp; 853 854 if (unlikely(sk_memalloc_socks())) 855 objp = __ac_get_obj(cachep, ac, flags, force_refill); 856 else 857 objp = ac->entry[--ac->avail]; 858 859 return objp; 860 } 861 862 static void *__ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac, 863 void *objp) 864 { 865 if (unlikely(pfmemalloc_active)) { 866 /* Some pfmemalloc slabs exist, check if this is one */ 867 struct page *page = virt_to_head_page(objp); 868 if (PageSlabPfmemalloc(page)) 869 set_obj_pfmemalloc(&objp); 870 } 871 872 return objp; 873 } 874 875 static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac, 876 void *objp) 877 { 878 if (unlikely(sk_memalloc_socks())) 879 objp = __ac_put_obj(cachep, ac, objp); 880 881 ac->entry[ac->avail++] = objp; 882 } 883 884 /* 885 * Transfer objects in one arraycache to another. 886 * Locking must be handled by the caller. 887 * 888 * Return the number of entries transferred. 889 */ 890 static int transfer_objects(struct array_cache *to, 891 struct array_cache *from, unsigned int max) 892 { 893 /* Figure out how many entries to transfer */ 894 int nr = min3(from->avail, max, to->limit - to->avail); 895 896 if (!nr) 897 return 0; 898 899 memcpy(to->entry + to->avail, from->entry + from->avail -nr, 900 sizeof(void *) *nr); 901 902 from->avail -= nr; 903 to->avail += nr; 904 return nr; 905 } 906 907 #ifndef CONFIG_NUMA 908 909 #define drain_alien_cache(cachep, alien) do { } while (0) 910 #define reap_alien(cachep, n) do { } while (0) 911 912 static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) 913 { 914 return (struct array_cache **)BAD_ALIEN_MAGIC; 915 } 916 917 static inline void free_alien_cache(struct array_cache **ac_ptr) 918 { 919 } 920 921 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 922 { 923 return 0; 924 } 925 926 static inline void *alternate_node_alloc(struct kmem_cache *cachep, 927 gfp_t flags) 928 { 929 return NULL; 930 } 931 932 static inline void *____cache_alloc_node(struct kmem_cache *cachep, 933 gfp_t flags, int nodeid) 934 { 935 return NULL; 936 } 937 938 #else /* CONFIG_NUMA */ 939 940 static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int); 941 static void *alternate_node_alloc(struct kmem_cache *, gfp_t); 942 943 static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp) 944 { 945 struct array_cache **ac_ptr; 946 int memsize = sizeof(void *) * nr_node_ids; 947 int i; 948 949 if (limit > 1) 950 limit = 12; 951 ac_ptr = kzalloc_node(memsize, gfp, node); 952 if (ac_ptr) { 953 for_each_node(i) { 954 if (i == node || !node_online(i)) 955 continue; 956 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp); 957 if (!ac_ptr[i]) { 958 for (i--; i >= 0; i--) 959 kfree(ac_ptr[i]); 960 kfree(ac_ptr); 961 return NULL; 962 } 963 } 964 } 965 return ac_ptr; 966 } 967 968 static void free_alien_cache(struct array_cache **ac_ptr) 969 { 970 int i; 971 972 if (!ac_ptr) 973 return; 974 for_each_node(i) 975 kfree(ac_ptr[i]); 976 kfree(ac_ptr); 977 } 978 979 static void __drain_alien_cache(struct kmem_cache *cachep, 980 struct array_cache *ac, int node) 981 { 982 struct kmem_cache_node *n = cachep->node[node]; 983 984 if (ac->avail) { 985 spin_lock(&n->list_lock); 986 /* 987 * Stuff objects into the remote nodes shared array first. 988 * That way we could avoid the overhead of putting the objects 989 * into the free lists and getting them back later. 990 */ 991 if (n->shared) 992 transfer_objects(n->shared, ac, ac->limit); 993 994 free_block(cachep, ac->entry, ac->avail, node); 995 ac->avail = 0; 996 spin_unlock(&n->list_lock); 997 } 998 } 999 1000 /* 1001 * Called from cache_reap() to regularly drain alien caches round robin. 1002 */ 1003 static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n) 1004 { 1005 int node = __this_cpu_read(slab_reap_node); 1006 1007 if (n->alien) { 1008 struct array_cache *ac = n->alien[node]; 1009 1010 if (ac && ac->avail && spin_trylock_irq(&ac->lock)) { 1011 __drain_alien_cache(cachep, ac, node); 1012 spin_unlock_irq(&ac->lock); 1013 } 1014 } 1015 } 1016 1017 static void drain_alien_cache(struct kmem_cache *cachep, 1018 struct array_cache **alien) 1019 { 1020 int i = 0; 1021 struct array_cache *ac; 1022 unsigned long flags; 1023 1024 for_each_online_node(i) { 1025 ac = alien[i]; 1026 if (ac) { 1027 spin_lock_irqsave(&ac->lock, flags); 1028 __drain_alien_cache(cachep, ac, i); 1029 spin_unlock_irqrestore(&ac->lock, flags); 1030 } 1031 } 1032 } 1033 1034 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp) 1035 { 1036 int nodeid = page_to_nid(virt_to_page(objp)); 1037 struct kmem_cache_node *n; 1038 struct array_cache *alien = NULL; 1039 int node; 1040 1041 node = numa_mem_id(); 1042 1043 /* 1044 * Make sure we are not freeing a object from another node to the array 1045 * cache on this cpu. 1046 */ 1047 if (likely(nodeid == node)) 1048 return 0; 1049 1050 n = cachep->node[node]; 1051 STATS_INC_NODEFREES(cachep); 1052 if (n->alien && n->alien[nodeid]) { 1053 alien = n->alien[nodeid]; 1054 spin_lock(&alien->lock); 1055 if (unlikely(alien->avail == alien->limit)) { 1056 STATS_INC_ACOVERFLOW(cachep); 1057 __drain_alien_cache(cachep, alien, nodeid); 1058 } 1059 ac_put_obj(cachep, alien, objp); 1060 spin_unlock(&alien->lock); 1061 } else { 1062 spin_lock(&(cachep->node[nodeid])->list_lock); 1063 free_block(cachep, &objp, 1, nodeid); 1064 spin_unlock(&(cachep->node[nodeid])->list_lock); 1065 } 1066 return 1; 1067 } 1068 #endif 1069 1070 /* 1071 * Allocates and initializes node for a node on each slab cache, used for 1072 * either memory or cpu hotplug. If memory is being hot-added, the kmem_cache_node 1073 * will be allocated off-node since memory is not yet online for the new node. 1074 * When hotplugging memory or a cpu, existing node are not replaced if 1075 * already in use. 1076 * 1077 * Must hold slab_mutex. 1078 */ 1079 static int init_cache_node_node(int node) 1080 { 1081 struct kmem_cache *cachep; 1082 struct kmem_cache_node *n; 1083 const int memsize = sizeof(struct kmem_cache_node); 1084 1085 list_for_each_entry(cachep, &slab_caches, list) { 1086 /* 1087 * Set up the kmem_cache_node for cpu before we can 1088 * begin anything. Make sure some other cpu on this 1089 * node has not already allocated this 1090 */ 1091 if (!cachep->node[node]) { 1092 n = kmalloc_node(memsize, GFP_KERNEL, node); 1093 if (!n) 1094 return -ENOMEM; 1095 kmem_cache_node_init(n); 1096 n->next_reap = jiffies + REAPTIMEOUT_NODE + 1097 ((unsigned long)cachep) % REAPTIMEOUT_NODE; 1098 1099 /* 1100 * The kmem_cache_nodes don't come and go as CPUs 1101 * come and go. slab_mutex is sufficient 1102 * protection here. 1103 */ 1104 cachep->node[node] = n; 1105 } 1106 1107 spin_lock_irq(&cachep->node[node]->list_lock); 1108 cachep->node[node]->free_limit = 1109 (1 + nr_cpus_node(node)) * 1110 cachep->batchcount + cachep->num; 1111 spin_unlock_irq(&cachep->node[node]->list_lock); 1112 } 1113 return 0; 1114 } 1115 1116 static inline int slabs_tofree(struct kmem_cache *cachep, 1117 struct kmem_cache_node *n) 1118 { 1119 return (n->free_objects + cachep->num - 1) / cachep->num; 1120 } 1121 1122 static void cpuup_canceled(long cpu) 1123 { 1124 struct kmem_cache *cachep; 1125 struct kmem_cache_node *n = NULL; 1126 int node = cpu_to_mem(cpu); 1127 const struct cpumask *mask = cpumask_of_node(node); 1128 1129 list_for_each_entry(cachep, &slab_caches, list) { 1130 struct array_cache *nc; 1131 struct array_cache *shared; 1132 struct array_cache **alien; 1133 1134 /* cpu is dead; no one can alloc from it. */ 1135 nc = cachep->array[cpu]; 1136 cachep->array[cpu] = NULL; 1137 n = cachep->node[node]; 1138 1139 if (!n) 1140 goto free_array_cache; 1141 1142 spin_lock_irq(&n->list_lock); 1143 1144 /* Free limit for this kmem_cache_node */ 1145 n->free_limit -= cachep->batchcount; 1146 if (nc) 1147 free_block(cachep, nc->entry, nc->avail, node); 1148 1149 if (!cpumask_empty(mask)) { 1150 spin_unlock_irq(&n->list_lock); 1151 goto free_array_cache; 1152 } 1153 1154 shared = n->shared; 1155 if (shared) { 1156 free_block(cachep, shared->entry, 1157 shared->avail, node); 1158 n->shared = NULL; 1159 } 1160 1161 alien = n->alien; 1162 n->alien = NULL; 1163 1164 spin_unlock_irq(&n->list_lock); 1165 1166 kfree(shared); 1167 if (alien) { 1168 drain_alien_cache(cachep, alien); 1169 free_alien_cache(alien); 1170 } 1171 free_array_cache: 1172 kfree(nc); 1173 } 1174 /* 1175 * In the previous loop, all the objects were freed to 1176 * the respective cache's slabs, now we can go ahead and 1177 * shrink each nodelist to its limit. 1178 */ 1179 list_for_each_entry(cachep, &slab_caches, list) { 1180 n = cachep->node[node]; 1181 if (!n) 1182 continue; 1183 drain_freelist(cachep, n, slabs_tofree(cachep, n)); 1184 } 1185 } 1186 1187 static int cpuup_prepare(long cpu) 1188 { 1189 struct kmem_cache *cachep; 1190 struct kmem_cache_node *n = NULL; 1191 int node = cpu_to_mem(cpu); 1192 int err; 1193 1194 /* 1195 * We need to do this right in the beginning since 1196 * alloc_arraycache's are going to use this list. 1197 * kmalloc_node allows us to add the slab to the right 1198 * kmem_cache_node and not this cpu's kmem_cache_node 1199 */ 1200 err = init_cache_node_node(node); 1201 if (err < 0) 1202 goto bad; 1203 1204 /* 1205 * Now we can go ahead with allocating the shared arrays and 1206 * array caches 1207 */ 1208 list_for_each_entry(cachep, &slab_caches, list) { 1209 struct array_cache *nc; 1210 struct array_cache *shared = NULL; 1211 struct array_cache **alien = NULL; 1212 1213 nc = alloc_arraycache(node, cachep->limit, 1214 cachep->batchcount, GFP_KERNEL); 1215 if (!nc) 1216 goto bad; 1217 if (cachep->shared) { 1218 shared = alloc_arraycache(node, 1219 cachep->shared * cachep->batchcount, 1220 0xbaadf00d, GFP_KERNEL); 1221 if (!shared) { 1222 kfree(nc); 1223 goto bad; 1224 } 1225 } 1226 if (use_alien_caches) { 1227 alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL); 1228 if (!alien) { 1229 kfree(shared); 1230 kfree(nc); 1231 goto bad; 1232 } 1233 } 1234 cachep->array[cpu] = nc; 1235 n = cachep->node[node]; 1236 BUG_ON(!n); 1237 1238 spin_lock_irq(&n->list_lock); 1239 if (!n->shared) { 1240 /* 1241 * We are serialised from CPU_DEAD or 1242 * CPU_UP_CANCELLED by the cpucontrol lock 1243 */ 1244 n->shared = shared; 1245 shared = NULL; 1246 } 1247 #ifdef CONFIG_NUMA 1248 if (!n->alien) { 1249 n->alien = alien; 1250 alien = NULL; 1251 } 1252 #endif 1253 spin_unlock_irq(&n->list_lock); 1254 kfree(shared); 1255 free_alien_cache(alien); 1256 if (cachep->flags & SLAB_DEBUG_OBJECTS) 1257 slab_set_debugobj_lock_classes_node(cachep, node); 1258 else if (!OFF_SLAB(cachep) && 1259 !(cachep->flags & SLAB_DESTROY_BY_RCU)) 1260 on_slab_lock_classes_node(cachep, node); 1261 } 1262 init_node_lock_keys(node); 1263 1264 return 0; 1265 bad: 1266 cpuup_canceled(cpu); 1267 return -ENOMEM; 1268 } 1269 1270 static int cpuup_callback(struct notifier_block *nfb, 1271 unsigned long action, void *hcpu) 1272 { 1273 long cpu = (long)hcpu; 1274 int err = 0; 1275 1276 switch (action) { 1277 case CPU_UP_PREPARE: 1278 case CPU_UP_PREPARE_FROZEN: 1279 mutex_lock(&slab_mutex); 1280 err = cpuup_prepare(cpu); 1281 mutex_unlock(&slab_mutex); 1282 break; 1283 case CPU_ONLINE: 1284 case CPU_ONLINE_FROZEN: 1285 start_cpu_timer(cpu); 1286 break; 1287 #ifdef CONFIG_HOTPLUG_CPU 1288 case CPU_DOWN_PREPARE: 1289 case CPU_DOWN_PREPARE_FROZEN: 1290 /* 1291 * Shutdown cache reaper. Note that the slab_mutex is 1292 * held so that if cache_reap() is invoked it cannot do 1293 * anything expensive but will only modify reap_work 1294 * and reschedule the timer. 1295 */ 1296 cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu)); 1297 /* Now the cache_reaper is guaranteed to be not running. */ 1298 per_cpu(slab_reap_work, cpu).work.func = NULL; 1299 break; 1300 case CPU_DOWN_FAILED: 1301 case CPU_DOWN_FAILED_FROZEN: 1302 start_cpu_timer(cpu); 1303 break; 1304 case CPU_DEAD: 1305 case CPU_DEAD_FROZEN: 1306 /* 1307 * Even if all the cpus of a node are down, we don't free the 1308 * kmem_cache_node of any cache. This to avoid a race between 1309 * cpu_down, and a kmalloc allocation from another cpu for 1310 * memory from the node of the cpu going down. The node 1311 * structure is usually allocated from kmem_cache_create() and 1312 * gets destroyed at kmem_cache_destroy(). 1313 */ 1314 /* fall through */ 1315 #endif 1316 case CPU_UP_CANCELED: 1317 case CPU_UP_CANCELED_FROZEN: 1318 mutex_lock(&slab_mutex); 1319 cpuup_canceled(cpu); 1320 mutex_unlock(&slab_mutex); 1321 break; 1322 } 1323 return notifier_from_errno(err); 1324 } 1325 1326 static struct notifier_block cpucache_notifier = { 1327 &cpuup_callback, NULL, 0 1328 }; 1329 1330 #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG) 1331 /* 1332 * Drains freelist for a node on each slab cache, used for memory hot-remove. 1333 * Returns -EBUSY if all objects cannot be drained so that the node is not 1334 * removed. 1335 * 1336 * Must hold slab_mutex. 1337 */ 1338 static int __meminit drain_cache_node_node(int node) 1339 { 1340 struct kmem_cache *cachep; 1341 int ret = 0; 1342 1343 list_for_each_entry(cachep, &slab_caches, list) { 1344 struct kmem_cache_node *n; 1345 1346 n = cachep->node[node]; 1347 if (!n) 1348 continue; 1349 1350 drain_freelist(cachep, n, slabs_tofree(cachep, n)); 1351 1352 if (!list_empty(&n->slabs_full) || 1353 !list_empty(&n->slabs_partial)) { 1354 ret = -EBUSY; 1355 break; 1356 } 1357 } 1358 return ret; 1359 } 1360 1361 static int __meminit slab_memory_callback(struct notifier_block *self, 1362 unsigned long action, void *arg) 1363 { 1364 struct memory_notify *mnb = arg; 1365 int ret = 0; 1366 int nid; 1367 1368 nid = mnb->status_change_nid; 1369 if (nid < 0) 1370 goto out; 1371 1372 switch (action) { 1373 case MEM_GOING_ONLINE: 1374 mutex_lock(&slab_mutex); 1375 ret = init_cache_node_node(nid); 1376 mutex_unlock(&slab_mutex); 1377 break; 1378 case MEM_GOING_OFFLINE: 1379 mutex_lock(&slab_mutex); 1380 ret = drain_cache_node_node(nid); 1381 mutex_unlock(&slab_mutex); 1382 break; 1383 case MEM_ONLINE: 1384 case MEM_OFFLINE: 1385 case MEM_CANCEL_ONLINE: 1386 case MEM_CANCEL_OFFLINE: 1387 break; 1388 } 1389 out: 1390 return notifier_from_errno(ret); 1391 } 1392 #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */ 1393 1394 /* 1395 * swap the static kmem_cache_node with kmalloced memory 1396 */ 1397 static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list, 1398 int nodeid) 1399 { 1400 struct kmem_cache_node *ptr; 1401 1402 ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid); 1403 BUG_ON(!ptr); 1404 1405 memcpy(ptr, list, sizeof(struct kmem_cache_node)); 1406 /* 1407 * Do not assume that spinlocks can be initialized via memcpy: 1408 */ 1409 spin_lock_init(&ptr->list_lock); 1410 1411 MAKE_ALL_LISTS(cachep, ptr, nodeid); 1412 cachep->node[nodeid] = ptr; 1413 } 1414 1415 /* 1416 * For setting up all the kmem_cache_node for cache whose buffer_size is same as 1417 * size of kmem_cache_node. 1418 */ 1419 static void __init set_up_node(struct kmem_cache *cachep, int index) 1420 { 1421 int node; 1422 1423 for_each_online_node(node) { 1424 cachep->node[node] = &init_kmem_cache_node[index + node]; 1425 cachep->node[node]->next_reap = jiffies + 1426 REAPTIMEOUT_NODE + 1427 ((unsigned long)cachep) % REAPTIMEOUT_NODE; 1428 } 1429 } 1430 1431 /* 1432 * The memory after the last cpu cache pointer is used for the 1433 * the node pointer. 1434 */ 1435 static void setup_node_pointer(struct kmem_cache *cachep) 1436 { 1437 cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids]; 1438 } 1439 1440 /* 1441 * Initialisation. Called after the page allocator have been initialised and 1442 * before smp_init(). 1443 */ 1444 void __init kmem_cache_init(void) 1445 { 1446 int i; 1447 1448 BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) < 1449 sizeof(struct rcu_head)); 1450 kmem_cache = &kmem_cache_boot; 1451 setup_node_pointer(kmem_cache); 1452 1453 if (num_possible_nodes() == 1) 1454 use_alien_caches = 0; 1455 1456 for (i = 0; i < NUM_INIT_LISTS; i++) 1457 kmem_cache_node_init(&init_kmem_cache_node[i]); 1458 1459 set_up_node(kmem_cache, CACHE_CACHE); 1460 1461 /* 1462 * Fragmentation resistance on low memory - only use bigger 1463 * page orders on machines with more than 32MB of memory if 1464 * not overridden on the command line. 1465 */ 1466 if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT) 1467 slab_max_order = SLAB_MAX_ORDER_HI; 1468 1469 /* Bootstrap is tricky, because several objects are allocated 1470 * from caches that do not exist yet: 1471 * 1) initialize the kmem_cache cache: it contains the struct 1472 * kmem_cache structures of all caches, except kmem_cache itself: 1473 * kmem_cache is statically allocated. 1474 * Initially an __init data area is used for the head array and the 1475 * kmem_cache_node structures, it's replaced with a kmalloc allocated 1476 * array at the end of the bootstrap. 1477 * 2) Create the first kmalloc cache. 1478 * The struct kmem_cache for the new cache is allocated normally. 1479 * An __init data area is used for the head array. 1480 * 3) Create the remaining kmalloc caches, with minimally sized 1481 * head arrays. 1482 * 4) Replace the __init data head arrays for kmem_cache and the first 1483 * kmalloc cache with kmalloc allocated arrays. 1484 * 5) Replace the __init data for kmem_cache_node for kmem_cache and 1485 * the other cache's with kmalloc allocated memory. 1486 * 6) Resize the head arrays of the kmalloc caches to their final sizes. 1487 */ 1488 1489 /* 1) create the kmem_cache */ 1490 1491 /* 1492 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids 1493 */ 1494 create_boot_cache(kmem_cache, "kmem_cache", 1495 offsetof(struct kmem_cache, array[nr_cpu_ids]) + 1496 nr_node_ids * sizeof(struct kmem_cache_node *), 1497 SLAB_HWCACHE_ALIGN); 1498 list_add(&kmem_cache->list, &slab_caches); 1499 1500 /* 2+3) create the kmalloc caches */ 1501 1502 /* 1503 * Initialize the caches that provide memory for the array cache and the 1504 * kmem_cache_node structures first. Without this, further allocations will 1505 * bug. 1506 */ 1507 1508 kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac", 1509 kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS); 1510 1511 if (INDEX_AC != INDEX_NODE) 1512 kmalloc_caches[INDEX_NODE] = 1513 create_kmalloc_cache("kmalloc-node", 1514 kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS); 1515 1516 slab_early_init = 0; 1517 1518 /* 4) Replace the bootstrap head arrays */ 1519 { 1520 struct array_cache *ptr; 1521 1522 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1523 1524 memcpy(ptr, cpu_cache_get(kmem_cache), 1525 sizeof(struct arraycache_init)); 1526 /* 1527 * Do not assume that spinlocks can be initialized via memcpy: 1528 */ 1529 spin_lock_init(&ptr->lock); 1530 1531 kmem_cache->array[smp_processor_id()] = ptr; 1532 1533 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1534 1535 BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC]) 1536 != &initarray_generic.cache); 1537 memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]), 1538 sizeof(struct arraycache_init)); 1539 /* 1540 * Do not assume that spinlocks can be initialized via memcpy: 1541 */ 1542 spin_lock_init(&ptr->lock); 1543 1544 kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr; 1545 } 1546 /* 5) Replace the bootstrap kmem_cache_node */ 1547 { 1548 int nid; 1549 1550 for_each_online_node(nid) { 1551 init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid); 1552 1553 init_list(kmalloc_caches[INDEX_AC], 1554 &init_kmem_cache_node[SIZE_AC + nid], nid); 1555 1556 if (INDEX_AC != INDEX_NODE) { 1557 init_list(kmalloc_caches[INDEX_NODE], 1558 &init_kmem_cache_node[SIZE_NODE + nid], nid); 1559 } 1560 } 1561 } 1562 1563 create_kmalloc_caches(ARCH_KMALLOC_FLAGS); 1564 } 1565 1566 void __init kmem_cache_init_late(void) 1567 { 1568 struct kmem_cache *cachep; 1569 1570 slab_state = UP; 1571 1572 /* 6) resize the head arrays to their final sizes */ 1573 mutex_lock(&slab_mutex); 1574 list_for_each_entry(cachep, &slab_caches, list) 1575 if (enable_cpucache(cachep, GFP_NOWAIT)) 1576 BUG(); 1577 mutex_unlock(&slab_mutex); 1578 1579 /* Annotate slab for lockdep -- annotate the malloc caches */ 1580 init_lock_keys(); 1581 1582 /* Done! */ 1583 slab_state = FULL; 1584 1585 /* 1586 * Register a cpu startup notifier callback that initializes 1587 * cpu_cache_get for all new cpus 1588 */ 1589 register_cpu_notifier(&cpucache_notifier); 1590 1591 #ifdef CONFIG_NUMA 1592 /* 1593 * Register a memory hotplug callback that initializes and frees 1594 * node. 1595 */ 1596 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 1597 #endif 1598 1599 /* 1600 * The reap timers are started later, with a module init call: That part 1601 * of the kernel is not yet operational. 1602 */ 1603 } 1604 1605 static int __init cpucache_init(void) 1606 { 1607 int cpu; 1608 1609 /* 1610 * Register the timers that return unneeded pages to the page allocator 1611 */ 1612 for_each_online_cpu(cpu) 1613 start_cpu_timer(cpu); 1614 1615 /* Done! */ 1616 slab_state = FULL; 1617 return 0; 1618 } 1619 __initcall(cpucache_init); 1620 1621 static noinline void 1622 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) 1623 { 1624 #if DEBUG 1625 struct kmem_cache_node *n; 1626 struct page *page; 1627 unsigned long flags; 1628 int node; 1629 static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 1630 DEFAULT_RATELIMIT_BURST); 1631 1632 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs)) 1633 return; 1634 1635 printk(KERN_WARNING 1636 "SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n", 1637 nodeid, gfpflags); 1638 printk(KERN_WARNING " cache: %s, object size: %d, order: %d\n", 1639 cachep->name, cachep->size, cachep->gfporder); 1640 1641 for_each_online_node(node) { 1642 unsigned long active_objs = 0, num_objs = 0, free_objects = 0; 1643 unsigned long active_slabs = 0, num_slabs = 0; 1644 1645 n = cachep->node[node]; 1646 if (!n) 1647 continue; 1648 1649 spin_lock_irqsave(&n->list_lock, flags); 1650 list_for_each_entry(page, &n->slabs_full, lru) { 1651 active_objs += cachep->num; 1652 active_slabs++; 1653 } 1654 list_for_each_entry(page, &n->slabs_partial, lru) { 1655 active_objs += page->active; 1656 active_slabs++; 1657 } 1658 list_for_each_entry(page, &n->slabs_free, lru) 1659 num_slabs++; 1660 1661 free_objects += n->free_objects; 1662 spin_unlock_irqrestore(&n->list_lock, flags); 1663 1664 num_slabs += active_slabs; 1665 num_objs = num_slabs * cachep->num; 1666 printk(KERN_WARNING 1667 " node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n", 1668 node, active_slabs, num_slabs, active_objs, num_objs, 1669 free_objects); 1670 } 1671 #endif 1672 } 1673 1674 /* 1675 * Interface to system's page allocator. No need to hold the cache-lock. 1676 * 1677 * If we requested dmaable memory, we will get it. Even if we 1678 * did not request dmaable memory, we might get it, but that 1679 * would be relatively rare and ignorable. 1680 */ 1681 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, 1682 int nodeid) 1683 { 1684 struct page *page; 1685 int nr_pages; 1686 1687 flags |= cachep->allocflags; 1688 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1689 flags |= __GFP_RECLAIMABLE; 1690 1691 if (memcg_charge_slab(cachep, flags, cachep->gfporder)) 1692 return NULL; 1693 1694 page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder); 1695 if (!page) { 1696 memcg_uncharge_slab(cachep, cachep->gfporder); 1697 slab_out_of_memory(cachep, flags, nodeid); 1698 return NULL; 1699 } 1700 1701 /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ 1702 if (unlikely(page->pfmemalloc)) 1703 pfmemalloc_active = true; 1704 1705 nr_pages = (1 << cachep->gfporder); 1706 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1707 add_zone_page_state(page_zone(page), 1708 NR_SLAB_RECLAIMABLE, nr_pages); 1709 else 1710 add_zone_page_state(page_zone(page), 1711 NR_SLAB_UNRECLAIMABLE, nr_pages); 1712 __SetPageSlab(page); 1713 if (page->pfmemalloc) 1714 SetPageSlabPfmemalloc(page); 1715 1716 if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) { 1717 kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid); 1718 1719 if (cachep->ctor) 1720 kmemcheck_mark_uninitialized_pages(page, nr_pages); 1721 else 1722 kmemcheck_mark_unallocated_pages(page, nr_pages); 1723 } 1724 1725 return page; 1726 } 1727 1728 /* 1729 * Interface to system's page release. 1730 */ 1731 static void kmem_freepages(struct kmem_cache *cachep, struct page *page) 1732 { 1733 const unsigned long nr_freed = (1 << cachep->gfporder); 1734 1735 kmemcheck_free_shadow(page, cachep->gfporder); 1736 1737 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1738 sub_zone_page_state(page_zone(page), 1739 NR_SLAB_RECLAIMABLE, nr_freed); 1740 else 1741 sub_zone_page_state(page_zone(page), 1742 NR_SLAB_UNRECLAIMABLE, nr_freed); 1743 1744 BUG_ON(!PageSlab(page)); 1745 __ClearPageSlabPfmemalloc(page); 1746 __ClearPageSlab(page); 1747 page_mapcount_reset(page); 1748 page->mapping = NULL; 1749 1750 if (current->reclaim_state) 1751 current->reclaim_state->reclaimed_slab += nr_freed; 1752 __free_pages(page, cachep->gfporder); 1753 memcg_uncharge_slab(cachep, cachep->gfporder); 1754 } 1755 1756 static void kmem_rcu_free(struct rcu_head *head) 1757 { 1758 struct kmem_cache *cachep; 1759 struct page *page; 1760 1761 page = container_of(head, struct page, rcu_head); 1762 cachep = page->slab_cache; 1763 1764 kmem_freepages(cachep, page); 1765 } 1766 1767 #if DEBUG 1768 1769 #ifdef CONFIG_DEBUG_PAGEALLOC 1770 static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr, 1771 unsigned long caller) 1772 { 1773 int size = cachep->object_size; 1774 1775 addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)]; 1776 1777 if (size < 5 * sizeof(unsigned long)) 1778 return; 1779 1780 *addr++ = 0x12345678; 1781 *addr++ = caller; 1782 *addr++ = smp_processor_id(); 1783 size -= 3 * sizeof(unsigned long); 1784 { 1785 unsigned long *sptr = &caller; 1786 unsigned long svalue; 1787 1788 while (!kstack_end(sptr)) { 1789 svalue = *sptr++; 1790 if (kernel_text_address(svalue)) { 1791 *addr++ = svalue; 1792 size -= sizeof(unsigned long); 1793 if (size <= sizeof(unsigned long)) 1794 break; 1795 } 1796 } 1797 1798 } 1799 *addr++ = 0x87654321; 1800 } 1801 #endif 1802 1803 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val) 1804 { 1805 int size = cachep->object_size; 1806 addr = &((char *)addr)[obj_offset(cachep)]; 1807 1808 memset(addr, val, size); 1809 *(unsigned char *)(addr + size - 1) = POISON_END; 1810 } 1811 1812 static void dump_line(char *data, int offset, int limit) 1813 { 1814 int i; 1815 unsigned char error = 0; 1816 int bad_count = 0; 1817 1818 printk(KERN_ERR "%03x: ", offset); 1819 for (i = 0; i < limit; i++) { 1820 if (data[offset + i] != POISON_FREE) { 1821 error = data[offset + i]; 1822 bad_count++; 1823 } 1824 } 1825 print_hex_dump(KERN_CONT, "", 0, 16, 1, 1826 &data[offset], limit, 1); 1827 1828 if (bad_count == 1) { 1829 error ^= POISON_FREE; 1830 if (!(error & (error - 1))) { 1831 printk(KERN_ERR "Single bit error detected. Probably " 1832 "bad RAM.\n"); 1833 #ifdef CONFIG_X86 1834 printk(KERN_ERR "Run memtest86+ or a similar memory " 1835 "test tool.\n"); 1836 #else 1837 printk(KERN_ERR "Run a memory test tool.\n"); 1838 #endif 1839 } 1840 } 1841 } 1842 #endif 1843 1844 #if DEBUG 1845 1846 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines) 1847 { 1848 int i, size; 1849 char *realobj; 1850 1851 if (cachep->flags & SLAB_RED_ZONE) { 1852 printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n", 1853 *dbg_redzone1(cachep, objp), 1854 *dbg_redzone2(cachep, objp)); 1855 } 1856 1857 if (cachep->flags & SLAB_STORE_USER) { 1858 printk(KERN_ERR "Last user: [<%p>](%pSR)\n", 1859 *dbg_userword(cachep, objp), 1860 *dbg_userword(cachep, objp)); 1861 } 1862 realobj = (char *)objp + obj_offset(cachep); 1863 size = cachep->object_size; 1864 for (i = 0; i < size && lines; i += 16, lines--) { 1865 int limit; 1866 limit = 16; 1867 if (i + limit > size) 1868 limit = size - i; 1869 dump_line(realobj, i, limit); 1870 } 1871 } 1872 1873 static void check_poison_obj(struct kmem_cache *cachep, void *objp) 1874 { 1875 char *realobj; 1876 int size, i; 1877 int lines = 0; 1878 1879 realobj = (char *)objp + obj_offset(cachep); 1880 size = cachep->object_size; 1881 1882 for (i = 0; i < size; i++) { 1883 char exp = POISON_FREE; 1884 if (i == size - 1) 1885 exp = POISON_END; 1886 if (realobj[i] != exp) { 1887 int limit; 1888 /* Mismatch ! */ 1889 /* Print header */ 1890 if (lines == 0) { 1891 printk(KERN_ERR 1892 "Slab corruption (%s): %s start=%p, len=%d\n", 1893 print_tainted(), cachep->name, realobj, size); 1894 print_objinfo(cachep, objp, 0); 1895 } 1896 /* Hexdump the affected line */ 1897 i = (i / 16) * 16; 1898 limit = 16; 1899 if (i + limit > size) 1900 limit = size - i; 1901 dump_line(realobj, i, limit); 1902 i += 16; 1903 lines++; 1904 /* Limit to 5 lines */ 1905 if (lines > 5) 1906 break; 1907 } 1908 } 1909 if (lines != 0) { 1910 /* Print some data about the neighboring objects, if they 1911 * exist: 1912 */ 1913 struct page *page = virt_to_head_page(objp); 1914 unsigned int objnr; 1915 1916 objnr = obj_to_index(cachep, page, objp); 1917 if (objnr) { 1918 objp = index_to_obj(cachep, page, objnr - 1); 1919 realobj = (char *)objp + obj_offset(cachep); 1920 printk(KERN_ERR "Prev obj: start=%p, len=%d\n", 1921 realobj, size); 1922 print_objinfo(cachep, objp, 2); 1923 } 1924 if (objnr + 1 < cachep->num) { 1925 objp = index_to_obj(cachep, page, objnr + 1); 1926 realobj = (char *)objp + obj_offset(cachep); 1927 printk(KERN_ERR "Next obj: start=%p, len=%d\n", 1928 realobj, size); 1929 print_objinfo(cachep, objp, 2); 1930 } 1931 } 1932 } 1933 #endif 1934 1935 #if DEBUG 1936 static void slab_destroy_debugcheck(struct kmem_cache *cachep, 1937 struct page *page) 1938 { 1939 int i; 1940 for (i = 0; i < cachep->num; i++) { 1941 void *objp = index_to_obj(cachep, page, i); 1942 1943 if (cachep->flags & SLAB_POISON) { 1944 #ifdef CONFIG_DEBUG_PAGEALLOC 1945 if (cachep->size % PAGE_SIZE == 0 && 1946 OFF_SLAB(cachep)) 1947 kernel_map_pages(virt_to_page(objp), 1948 cachep->size / PAGE_SIZE, 1); 1949 else 1950 check_poison_obj(cachep, objp); 1951 #else 1952 check_poison_obj(cachep, objp); 1953 #endif 1954 } 1955 if (cachep->flags & SLAB_RED_ZONE) { 1956 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 1957 slab_error(cachep, "start of a freed object " 1958 "was overwritten"); 1959 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 1960 slab_error(cachep, "end of a freed object " 1961 "was overwritten"); 1962 } 1963 } 1964 } 1965 #else 1966 static void slab_destroy_debugcheck(struct kmem_cache *cachep, 1967 struct page *page) 1968 { 1969 } 1970 #endif 1971 1972 /** 1973 * slab_destroy - destroy and release all objects in a slab 1974 * @cachep: cache pointer being destroyed 1975 * @page: page pointer being destroyed 1976 * 1977 * Destroy all the objs in a slab, and release the mem back to the system. 1978 * Before calling the slab must have been unlinked from the cache. The 1979 * cache-lock is not held/needed. 1980 */ 1981 static void slab_destroy(struct kmem_cache *cachep, struct page *page) 1982 { 1983 void *freelist; 1984 1985 freelist = page->freelist; 1986 slab_destroy_debugcheck(cachep, page); 1987 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { 1988 struct rcu_head *head; 1989 1990 /* 1991 * RCU free overloads the RCU head over the LRU. 1992 * slab_page has been overloeaded over the LRU, 1993 * however it is not used from now on so that 1994 * we can use it safely. 1995 */ 1996 head = (void *)&page->rcu_head; 1997 call_rcu(head, kmem_rcu_free); 1998 1999 } else { 2000 kmem_freepages(cachep, page); 2001 } 2002 2003 /* 2004 * From now on, we don't use freelist 2005 * although actual page can be freed in rcu context 2006 */ 2007 if (OFF_SLAB(cachep)) 2008 kmem_cache_free(cachep->freelist_cache, freelist); 2009 } 2010 2011 /** 2012 * calculate_slab_order - calculate size (page order) of slabs 2013 * @cachep: pointer to the cache that is being created 2014 * @size: size of objects to be created in this cache. 2015 * @align: required alignment for the objects. 2016 * @flags: slab allocation flags 2017 * 2018 * Also calculates the number of objects per slab. 2019 * 2020 * This could be made much more intelligent. For now, try to avoid using 2021 * high order pages for slabs. When the gfp() functions are more friendly 2022 * towards high-order requests, this should be changed. 2023 */ 2024 static size_t calculate_slab_order(struct kmem_cache *cachep, 2025 size_t size, size_t align, unsigned long flags) 2026 { 2027 unsigned long offslab_limit; 2028 size_t left_over = 0; 2029 int gfporder; 2030 2031 for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) { 2032 unsigned int num; 2033 size_t remainder; 2034 2035 cache_estimate(gfporder, size, align, flags, &remainder, &num); 2036 if (!num) 2037 continue; 2038 2039 /* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */ 2040 if (num > SLAB_OBJ_MAX_NUM) 2041 break; 2042 2043 if (flags & CFLGS_OFF_SLAB) { 2044 /* 2045 * Max number of objs-per-slab for caches which 2046 * use off-slab slabs. Needed to avoid a possible 2047 * looping condition in cache_grow(). 2048 */ 2049 offslab_limit = size; 2050 offslab_limit /= sizeof(freelist_idx_t); 2051 2052 if (num > offslab_limit) 2053 break; 2054 } 2055 2056 /* Found something acceptable - save it away */ 2057 cachep->num = num; 2058 cachep->gfporder = gfporder; 2059 left_over = remainder; 2060 2061 /* 2062 * A VFS-reclaimable slab tends to have most allocations 2063 * as GFP_NOFS and we really don't want to have to be allocating 2064 * higher-order pages when we are unable to shrink dcache. 2065 */ 2066 if (flags & SLAB_RECLAIM_ACCOUNT) 2067 break; 2068 2069 /* 2070 * Large number of objects is good, but very large slabs are 2071 * currently bad for the gfp()s. 2072 */ 2073 if (gfporder >= slab_max_order) 2074 break; 2075 2076 /* 2077 * Acceptable internal fragmentation? 2078 */ 2079 if (left_over * 8 <= (PAGE_SIZE << gfporder)) 2080 break; 2081 } 2082 return left_over; 2083 } 2084 2085 static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp) 2086 { 2087 if (slab_state >= FULL) 2088 return enable_cpucache(cachep, gfp); 2089 2090 if (slab_state == DOWN) { 2091 /* 2092 * Note: Creation of first cache (kmem_cache). 2093 * The setup_node is taken care 2094 * of by the caller of __kmem_cache_create 2095 */ 2096 cachep->array[smp_processor_id()] = &initarray_generic.cache; 2097 slab_state = PARTIAL; 2098 } else if (slab_state == PARTIAL) { 2099 /* 2100 * Note: the second kmem_cache_create must create the cache 2101 * that's used by kmalloc(24), otherwise the creation of 2102 * further caches will BUG(). 2103 */ 2104 cachep->array[smp_processor_id()] = &initarray_generic.cache; 2105 2106 /* 2107 * If the cache that's used by kmalloc(sizeof(kmem_cache_node)) is 2108 * the second cache, then we need to set up all its node/, 2109 * otherwise the creation of further caches will BUG(). 2110 */ 2111 set_up_node(cachep, SIZE_AC); 2112 if (INDEX_AC == INDEX_NODE) 2113 slab_state = PARTIAL_NODE; 2114 else 2115 slab_state = PARTIAL_ARRAYCACHE; 2116 } else { 2117 /* Remaining boot caches */ 2118 cachep->array[smp_processor_id()] = 2119 kmalloc(sizeof(struct arraycache_init), gfp); 2120 2121 if (slab_state == PARTIAL_ARRAYCACHE) { 2122 set_up_node(cachep, SIZE_NODE); 2123 slab_state = PARTIAL_NODE; 2124 } else { 2125 int node; 2126 for_each_online_node(node) { 2127 cachep->node[node] = 2128 kmalloc_node(sizeof(struct kmem_cache_node), 2129 gfp, node); 2130 BUG_ON(!cachep->node[node]); 2131 kmem_cache_node_init(cachep->node[node]); 2132 } 2133 } 2134 } 2135 cachep->node[numa_mem_id()]->next_reap = 2136 jiffies + REAPTIMEOUT_NODE + 2137 ((unsigned long)cachep) % REAPTIMEOUT_NODE; 2138 2139 cpu_cache_get(cachep)->avail = 0; 2140 cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; 2141 cpu_cache_get(cachep)->batchcount = 1; 2142 cpu_cache_get(cachep)->touched = 0; 2143 cachep->batchcount = 1; 2144 cachep->limit = BOOT_CPUCACHE_ENTRIES; 2145 return 0; 2146 } 2147 2148 /** 2149 * __kmem_cache_create - Create a cache. 2150 * @cachep: cache management descriptor 2151 * @flags: SLAB flags 2152 * 2153 * Returns a ptr to the cache on success, NULL on failure. 2154 * Cannot be called within a int, but can be interrupted. 2155 * The @ctor is run when new pages are allocated by the cache. 2156 * 2157 * The flags are 2158 * 2159 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 2160 * to catch references to uninitialised memory. 2161 * 2162 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 2163 * for buffer overruns. 2164 * 2165 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 2166 * cacheline. This can be beneficial if you're counting cycles as closely 2167 * as davem. 2168 */ 2169 int 2170 __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags) 2171 { 2172 size_t left_over, freelist_size, ralign; 2173 gfp_t gfp; 2174 int err; 2175 size_t size = cachep->size; 2176 2177 #if DEBUG 2178 #if FORCED_DEBUG 2179 /* 2180 * Enable redzoning and last user accounting, except for caches with 2181 * large objects, if the increased size would increase the object size 2182 * above the next power of two: caches with object sizes just above a 2183 * power of two have a significant amount of internal fragmentation. 2184 */ 2185 if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN + 2186 2 * sizeof(unsigned long long))) 2187 flags |= SLAB_RED_ZONE | SLAB_STORE_USER; 2188 if (!(flags & SLAB_DESTROY_BY_RCU)) 2189 flags |= SLAB_POISON; 2190 #endif 2191 if (flags & SLAB_DESTROY_BY_RCU) 2192 BUG_ON(flags & SLAB_POISON); 2193 #endif 2194 2195 /* 2196 * Check that size is in terms of words. This is needed to avoid 2197 * unaligned accesses for some archs when redzoning is used, and makes 2198 * sure any on-slab bufctl's are also correctly aligned. 2199 */ 2200 if (size & (BYTES_PER_WORD - 1)) { 2201 size += (BYTES_PER_WORD - 1); 2202 size &= ~(BYTES_PER_WORD - 1); 2203 } 2204 2205 /* 2206 * Redzoning and user store require word alignment or possibly larger. 2207 * Note this will be overridden by architecture or caller mandated 2208 * alignment if either is greater than BYTES_PER_WORD. 2209 */ 2210 if (flags & SLAB_STORE_USER) 2211 ralign = BYTES_PER_WORD; 2212 2213 if (flags & SLAB_RED_ZONE) { 2214 ralign = REDZONE_ALIGN; 2215 /* If redzoning, ensure that the second redzone is suitably 2216 * aligned, by adjusting the object size accordingly. */ 2217 size += REDZONE_ALIGN - 1; 2218 size &= ~(REDZONE_ALIGN - 1); 2219 } 2220 2221 /* 3) caller mandated alignment */ 2222 if (ralign < cachep->align) { 2223 ralign = cachep->align; 2224 } 2225 /* disable debug if necessary */ 2226 if (ralign > __alignof__(unsigned long long)) 2227 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2228 /* 2229 * 4) Store it. 2230 */ 2231 cachep->align = ralign; 2232 2233 if (slab_is_available()) 2234 gfp = GFP_KERNEL; 2235 else 2236 gfp = GFP_NOWAIT; 2237 2238 setup_node_pointer(cachep); 2239 #if DEBUG 2240 2241 /* 2242 * Both debugging options require word-alignment which is calculated 2243 * into align above. 2244 */ 2245 if (flags & SLAB_RED_ZONE) { 2246 /* add space for red zone words */ 2247 cachep->obj_offset += sizeof(unsigned long long); 2248 size += 2 * sizeof(unsigned long long); 2249 } 2250 if (flags & SLAB_STORE_USER) { 2251 /* user store requires one word storage behind the end of 2252 * the real object. But if the second red zone needs to be 2253 * aligned to 64 bits, we must allow that much space. 2254 */ 2255 if (flags & SLAB_RED_ZONE) 2256 size += REDZONE_ALIGN; 2257 else 2258 size += BYTES_PER_WORD; 2259 } 2260 #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) 2261 if (size >= kmalloc_size(INDEX_NODE + 1) 2262 && cachep->object_size > cache_line_size() 2263 && ALIGN(size, cachep->align) < PAGE_SIZE) { 2264 cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align); 2265 size = PAGE_SIZE; 2266 } 2267 #endif 2268 #endif 2269 2270 /* 2271 * Determine if the slab management is 'on' or 'off' slab. 2272 * (bootstrapping cannot cope with offslab caches so don't do 2273 * it too early on. Always use on-slab management when 2274 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak) 2275 */ 2276 if ((size >= (PAGE_SIZE >> 5)) && !slab_early_init && 2277 !(flags & SLAB_NOLEAKTRACE)) 2278 /* 2279 * Size is large, assume best to place the slab management obj 2280 * off-slab (should allow better packing of objs). 2281 */ 2282 flags |= CFLGS_OFF_SLAB; 2283 2284 size = ALIGN(size, cachep->align); 2285 /* 2286 * We should restrict the number of objects in a slab to implement 2287 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition. 2288 */ 2289 if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE) 2290 size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align); 2291 2292 left_over = calculate_slab_order(cachep, size, cachep->align, flags); 2293 2294 if (!cachep->num) 2295 return -E2BIG; 2296 2297 freelist_size = 2298 ALIGN(cachep->num * sizeof(freelist_idx_t), cachep->align); 2299 2300 /* 2301 * If the slab has been placed off-slab, and we have enough space then 2302 * move it on-slab. This is at the expense of any extra colouring. 2303 */ 2304 if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) { 2305 flags &= ~CFLGS_OFF_SLAB; 2306 left_over -= freelist_size; 2307 } 2308 2309 if (flags & CFLGS_OFF_SLAB) { 2310 /* really off slab. No need for manual alignment */ 2311 freelist_size = cachep->num * sizeof(freelist_idx_t); 2312 2313 #ifdef CONFIG_PAGE_POISONING 2314 /* If we're going to use the generic kernel_map_pages() 2315 * poisoning, then it's going to smash the contents of 2316 * the redzone and userword anyhow, so switch them off. 2317 */ 2318 if (size % PAGE_SIZE == 0 && flags & SLAB_POISON) 2319 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); 2320 #endif 2321 } 2322 2323 cachep->colour_off = cache_line_size(); 2324 /* Offset must be a multiple of the alignment. */ 2325 if (cachep->colour_off < cachep->align) 2326 cachep->colour_off = cachep->align; 2327 cachep->colour = left_over / cachep->colour_off; 2328 cachep->freelist_size = freelist_size; 2329 cachep->flags = flags; 2330 cachep->allocflags = __GFP_COMP; 2331 if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA)) 2332 cachep->allocflags |= GFP_DMA; 2333 cachep->size = size; 2334 cachep->reciprocal_buffer_size = reciprocal_value(size); 2335 2336 if (flags & CFLGS_OFF_SLAB) { 2337 cachep->freelist_cache = kmalloc_slab(freelist_size, 0u); 2338 /* 2339 * This is a possibility for one of the kmalloc_{dma,}_caches. 2340 * But since we go off slab only for object size greater than 2341 * PAGE_SIZE/8, and kmalloc_{dma,}_caches get created 2342 * in ascending order,this should not happen at all. 2343 * But leave a BUG_ON for some lucky dude. 2344 */ 2345 BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache)); 2346 } 2347 2348 err = setup_cpu_cache(cachep, gfp); 2349 if (err) { 2350 __kmem_cache_shutdown(cachep); 2351 return err; 2352 } 2353 2354 if (flags & SLAB_DEBUG_OBJECTS) { 2355 /* 2356 * Would deadlock through slab_destroy()->call_rcu()-> 2357 * debug_object_activate()->kmem_cache_alloc(). 2358 */ 2359 WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU); 2360 2361 slab_set_debugobj_lock_classes(cachep); 2362 } else if (!OFF_SLAB(cachep) && !(flags & SLAB_DESTROY_BY_RCU)) 2363 on_slab_lock_classes(cachep); 2364 2365 return 0; 2366 } 2367 2368 #if DEBUG 2369 static void check_irq_off(void) 2370 { 2371 BUG_ON(!irqs_disabled()); 2372 } 2373 2374 static void check_irq_on(void) 2375 { 2376 BUG_ON(irqs_disabled()); 2377 } 2378 2379 static void check_spinlock_acquired(struct kmem_cache *cachep) 2380 { 2381 #ifdef CONFIG_SMP 2382 check_irq_off(); 2383 assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock); 2384 #endif 2385 } 2386 2387 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node) 2388 { 2389 #ifdef CONFIG_SMP 2390 check_irq_off(); 2391 assert_spin_locked(&cachep->node[node]->list_lock); 2392 #endif 2393 } 2394 2395 #else 2396 #define check_irq_off() do { } while(0) 2397 #define check_irq_on() do { } while(0) 2398 #define check_spinlock_acquired(x) do { } while(0) 2399 #define check_spinlock_acquired_node(x, y) do { } while(0) 2400 #endif 2401 2402 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, 2403 struct array_cache *ac, 2404 int force, int node); 2405 2406 static void do_drain(void *arg) 2407 { 2408 struct kmem_cache *cachep = arg; 2409 struct array_cache *ac; 2410 int node = numa_mem_id(); 2411 2412 check_irq_off(); 2413 ac = cpu_cache_get(cachep); 2414 spin_lock(&cachep->node[node]->list_lock); 2415 free_block(cachep, ac->entry, ac->avail, node); 2416 spin_unlock(&cachep->node[node]->list_lock); 2417 ac->avail = 0; 2418 } 2419 2420 static void drain_cpu_caches(struct kmem_cache *cachep) 2421 { 2422 struct kmem_cache_node *n; 2423 int node; 2424 2425 on_each_cpu(do_drain, cachep, 1); 2426 check_irq_on(); 2427 for_each_online_node(node) { 2428 n = cachep->node[node]; 2429 if (n && n->alien) 2430 drain_alien_cache(cachep, n->alien); 2431 } 2432 2433 for_each_online_node(node) { 2434 n = cachep->node[node]; 2435 if (n) 2436 drain_array(cachep, n, n->shared, 1, node); 2437 } 2438 } 2439 2440 /* 2441 * Remove slabs from the list of free slabs. 2442 * Specify the number of slabs to drain in tofree. 2443 * 2444 * Returns the actual number of slabs released. 2445 */ 2446 static int drain_freelist(struct kmem_cache *cache, 2447 struct kmem_cache_node *n, int tofree) 2448 { 2449 struct list_head *p; 2450 int nr_freed; 2451 struct page *page; 2452 2453 nr_freed = 0; 2454 while (nr_freed < tofree && !list_empty(&n->slabs_free)) { 2455 2456 spin_lock_irq(&n->list_lock); 2457 p = n->slabs_free.prev; 2458 if (p == &n->slabs_free) { 2459 spin_unlock_irq(&n->list_lock); 2460 goto out; 2461 } 2462 2463 page = list_entry(p, struct page, lru); 2464 #if DEBUG 2465 BUG_ON(page->active); 2466 #endif 2467 list_del(&page->lru); 2468 /* 2469 * Safe to drop the lock. The slab is no longer linked 2470 * to the cache. 2471 */ 2472 n->free_objects -= cache->num; 2473 spin_unlock_irq(&n->list_lock); 2474 slab_destroy(cache, page); 2475 nr_freed++; 2476 } 2477 out: 2478 return nr_freed; 2479 } 2480 2481 int __kmem_cache_shrink(struct kmem_cache *cachep) 2482 { 2483 int ret = 0, i = 0; 2484 struct kmem_cache_node *n; 2485 2486 drain_cpu_caches(cachep); 2487 2488 check_irq_on(); 2489 for_each_online_node(i) { 2490 n = cachep->node[i]; 2491 if (!n) 2492 continue; 2493 2494 drain_freelist(cachep, n, slabs_tofree(cachep, n)); 2495 2496 ret += !list_empty(&n->slabs_full) || 2497 !list_empty(&n->slabs_partial); 2498 } 2499 return (ret ? 1 : 0); 2500 } 2501 2502 int __kmem_cache_shutdown(struct kmem_cache *cachep) 2503 { 2504 int i; 2505 struct kmem_cache_node *n; 2506 int rc = __kmem_cache_shrink(cachep); 2507 2508 if (rc) 2509 return rc; 2510 2511 for_each_online_cpu(i) 2512 kfree(cachep->array[i]); 2513 2514 /* NUMA: free the node structures */ 2515 for_each_online_node(i) { 2516 n = cachep->node[i]; 2517 if (n) { 2518 kfree(n->shared); 2519 free_alien_cache(n->alien); 2520 kfree(n); 2521 } 2522 } 2523 return 0; 2524 } 2525 2526 /* 2527 * Get the memory for a slab management obj. 2528 * 2529 * For a slab cache when the slab descriptor is off-slab, the 2530 * slab descriptor can't come from the same cache which is being created, 2531 * Because if it is the case, that means we defer the creation of 2532 * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point. 2533 * And we eventually call down to __kmem_cache_create(), which 2534 * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one. 2535 * This is a "chicken-and-egg" problem. 2536 * 2537 * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches, 2538 * which are all initialized during kmem_cache_init(). 2539 */ 2540 static void *alloc_slabmgmt(struct kmem_cache *cachep, 2541 struct page *page, int colour_off, 2542 gfp_t local_flags, int nodeid) 2543 { 2544 void *freelist; 2545 void *addr = page_address(page); 2546 2547 if (OFF_SLAB(cachep)) { 2548 /* Slab management obj is off-slab. */ 2549 freelist = kmem_cache_alloc_node(cachep->freelist_cache, 2550 local_flags, nodeid); 2551 if (!freelist) 2552 return NULL; 2553 } else { 2554 freelist = addr + colour_off; 2555 colour_off += cachep->freelist_size; 2556 } 2557 page->active = 0; 2558 page->s_mem = addr + colour_off; 2559 return freelist; 2560 } 2561 2562 static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx) 2563 { 2564 return ((freelist_idx_t *)page->freelist)[idx]; 2565 } 2566 2567 static inline void set_free_obj(struct page *page, 2568 unsigned int idx, freelist_idx_t val) 2569 { 2570 ((freelist_idx_t *)(page->freelist))[idx] = val; 2571 } 2572 2573 static void cache_init_objs(struct kmem_cache *cachep, 2574 struct page *page) 2575 { 2576 int i; 2577 2578 for (i = 0; i < cachep->num; i++) { 2579 void *objp = index_to_obj(cachep, page, i); 2580 #if DEBUG 2581 /* need to poison the objs? */ 2582 if (cachep->flags & SLAB_POISON) 2583 poison_obj(cachep, objp, POISON_FREE); 2584 if (cachep->flags & SLAB_STORE_USER) 2585 *dbg_userword(cachep, objp) = NULL; 2586 2587 if (cachep->flags & SLAB_RED_ZONE) { 2588 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2589 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2590 } 2591 /* 2592 * Constructors are not allowed to allocate memory from the same 2593 * cache which they are a constructor for. Otherwise, deadlock. 2594 * They must also be threaded. 2595 */ 2596 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) 2597 cachep->ctor(objp + obj_offset(cachep)); 2598 2599 if (cachep->flags & SLAB_RED_ZONE) { 2600 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 2601 slab_error(cachep, "constructor overwrote the" 2602 " end of an object"); 2603 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 2604 slab_error(cachep, "constructor overwrote the" 2605 " start of an object"); 2606 } 2607 if ((cachep->size % PAGE_SIZE) == 0 && 2608 OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) 2609 kernel_map_pages(virt_to_page(objp), 2610 cachep->size / PAGE_SIZE, 0); 2611 #else 2612 if (cachep->ctor) 2613 cachep->ctor(objp); 2614 #endif 2615 set_free_obj(page, i, i); 2616 } 2617 } 2618 2619 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags) 2620 { 2621 if (CONFIG_ZONE_DMA_FLAG) { 2622 if (flags & GFP_DMA) 2623 BUG_ON(!(cachep->allocflags & GFP_DMA)); 2624 else 2625 BUG_ON(cachep->allocflags & GFP_DMA); 2626 } 2627 } 2628 2629 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page, 2630 int nodeid) 2631 { 2632 void *objp; 2633 2634 objp = index_to_obj(cachep, page, get_free_obj(page, page->active)); 2635 page->active++; 2636 #if DEBUG 2637 WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); 2638 #endif 2639 2640 return objp; 2641 } 2642 2643 static void slab_put_obj(struct kmem_cache *cachep, struct page *page, 2644 void *objp, int nodeid) 2645 { 2646 unsigned int objnr = obj_to_index(cachep, page, objp); 2647 #if DEBUG 2648 unsigned int i; 2649 2650 /* Verify that the slab belongs to the intended node */ 2651 WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid); 2652 2653 /* Verify double free bug */ 2654 for (i = page->active; i < cachep->num; i++) { 2655 if (get_free_obj(page, i) == objnr) { 2656 printk(KERN_ERR "slab: double free detected in cache " 2657 "'%s', objp %p\n", cachep->name, objp); 2658 BUG(); 2659 } 2660 } 2661 #endif 2662 page->active--; 2663 set_free_obj(page, page->active, objnr); 2664 } 2665 2666 /* 2667 * Map pages beginning at addr to the given cache and slab. This is required 2668 * for the slab allocator to be able to lookup the cache and slab of a 2669 * virtual address for kfree, ksize, and slab debugging. 2670 */ 2671 static void slab_map_pages(struct kmem_cache *cache, struct page *page, 2672 void *freelist) 2673 { 2674 page->slab_cache = cache; 2675 page->freelist = freelist; 2676 } 2677 2678 /* 2679 * Grow (by 1) the number of slabs within a cache. This is called by 2680 * kmem_cache_alloc() when there are no active objs left in a cache. 2681 */ 2682 static int cache_grow(struct kmem_cache *cachep, 2683 gfp_t flags, int nodeid, struct page *page) 2684 { 2685 void *freelist; 2686 size_t offset; 2687 gfp_t local_flags; 2688 struct kmem_cache_node *n; 2689 2690 /* 2691 * Be lazy and only check for valid flags here, keeping it out of the 2692 * critical path in kmem_cache_alloc(). 2693 */ 2694 BUG_ON(flags & GFP_SLAB_BUG_MASK); 2695 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); 2696 2697 /* Take the node list lock to change the colour_next on this node */ 2698 check_irq_off(); 2699 n = cachep->node[nodeid]; 2700 spin_lock(&n->list_lock); 2701 2702 /* Get colour for the slab, and cal the next value. */ 2703 offset = n->colour_next; 2704 n->colour_next++; 2705 if (n->colour_next >= cachep->colour) 2706 n->colour_next = 0; 2707 spin_unlock(&n->list_lock); 2708 2709 offset *= cachep->colour_off; 2710 2711 if (local_flags & __GFP_WAIT) 2712 local_irq_enable(); 2713 2714 /* 2715 * The test for missing atomic flag is performed here, rather than 2716 * the more obvious place, simply to reduce the critical path length 2717 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they 2718 * will eventually be caught here (where it matters). 2719 */ 2720 kmem_flagcheck(cachep, flags); 2721 2722 /* 2723 * Get mem for the objs. Attempt to allocate a physical page from 2724 * 'nodeid'. 2725 */ 2726 if (!page) 2727 page = kmem_getpages(cachep, local_flags, nodeid); 2728 if (!page) 2729 goto failed; 2730 2731 /* Get slab management. */ 2732 freelist = alloc_slabmgmt(cachep, page, offset, 2733 local_flags & ~GFP_CONSTRAINT_MASK, nodeid); 2734 if (!freelist) 2735 goto opps1; 2736 2737 slab_map_pages(cachep, page, freelist); 2738 2739 cache_init_objs(cachep, page); 2740 2741 if (local_flags & __GFP_WAIT) 2742 local_irq_disable(); 2743 check_irq_off(); 2744 spin_lock(&n->list_lock); 2745 2746 /* Make slab active. */ 2747 list_add_tail(&page->lru, &(n->slabs_free)); 2748 STATS_INC_GROWN(cachep); 2749 n->free_objects += cachep->num; 2750 spin_unlock(&n->list_lock); 2751 return 1; 2752 opps1: 2753 kmem_freepages(cachep, page); 2754 failed: 2755 if (local_flags & __GFP_WAIT) 2756 local_irq_disable(); 2757 return 0; 2758 } 2759 2760 #if DEBUG 2761 2762 /* 2763 * Perform extra freeing checks: 2764 * - detect bad pointers. 2765 * - POISON/RED_ZONE checking 2766 */ 2767 static void kfree_debugcheck(const void *objp) 2768 { 2769 if (!virt_addr_valid(objp)) { 2770 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", 2771 (unsigned long)objp); 2772 BUG(); 2773 } 2774 } 2775 2776 static inline void verify_redzone_free(struct kmem_cache *cache, void *obj) 2777 { 2778 unsigned long long redzone1, redzone2; 2779 2780 redzone1 = *dbg_redzone1(cache, obj); 2781 redzone2 = *dbg_redzone2(cache, obj); 2782 2783 /* 2784 * Redzone is ok. 2785 */ 2786 if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE) 2787 return; 2788 2789 if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE) 2790 slab_error(cache, "double free detected"); 2791 else 2792 slab_error(cache, "memory outside object was overwritten"); 2793 2794 printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n", 2795 obj, redzone1, redzone2); 2796 } 2797 2798 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp, 2799 unsigned long caller) 2800 { 2801 unsigned int objnr; 2802 struct page *page; 2803 2804 BUG_ON(virt_to_cache(objp) != cachep); 2805 2806 objp -= obj_offset(cachep); 2807 kfree_debugcheck(objp); 2808 page = virt_to_head_page(objp); 2809 2810 if (cachep->flags & SLAB_RED_ZONE) { 2811 verify_redzone_free(cachep, objp); 2812 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2813 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2814 } 2815 if (cachep->flags & SLAB_STORE_USER) 2816 *dbg_userword(cachep, objp) = (void *)caller; 2817 2818 objnr = obj_to_index(cachep, page, objp); 2819 2820 BUG_ON(objnr >= cachep->num); 2821 BUG_ON(objp != index_to_obj(cachep, page, objnr)); 2822 2823 if (cachep->flags & SLAB_POISON) { 2824 #ifdef CONFIG_DEBUG_PAGEALLOC 2825 if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) { 2826 store_stackinfo(cachep, objp, caller); 2827 kernel_map_pages(virt_to_page(objp), 2828 cachep->size / PAGE_SIZE, 0); 2829 } else { 2830 poison_obj(cachep, objp, POISON_FREE); 2831 } 2832 #else 2833 poison_obj(cachep, objp, POISON_FREE); 2834 #endif 2835 } 2836 return objp; 2837 } 2838 2839 #else 2840 #define kfree_debugcheck(x) do { } while(0) 2841 #define cache_free_debugcheck(x,objp,z) (objp) 2842 #endif 2843 2844 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags, 2845 bool force_refill) 2846 { 2847 int batchcount; 2848 struct kmem_cache_node *n; 2849 struct array_cache *ac; 2850 int node; 2851 2852 check_irq_off(); 2853 node = numa_mem_id(); 2854 if (unlikely(force_refill)) 2855 goto force_grow; 2856 retry: 2857 ac = cpu_cache_get(cachep); 2858 batchcount = ac->batchcount; 2859 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { 2860 /* 2861 * If there was little recent activity on this cache, then 2862 * perform only a partial refill. Otherwise we could generate 2863 * refill bouncing. 2864 */ 2865 batchcount = BATCHREFILL_LIMIT; 2866 } 2867 n = cachep->node[node]; 2868 2869 BUG_ON(ac->avail > 0 || !n); 2870 spin_lock(&n->list_lock); 2871 2872 /* See if we can refill from the shared array */ 2873 if (n->shared && transfer_objects(ac, n->shared, batchcount)) { 2874 n->shared->touched = 1; 2875 goto alloc_done; 2876 } 2877 2878 while (batchcount > 0) { 2879 struct list_head *entry; 2880 struct page *page; 2881 /* Get slab alloc is to come from. */ 2882 entry = n->slabs_partial.next; 2883 if (entry == &n->slabs_partial) { 2884 n->free_touched = 1; 2885 entry = n->slabs_free.next; 2886 if (entry == &n->slabs_free) 2887 goto must_grow; 2888 } 2889 2890 page = list_entry(entry, struct page, lru); 2891 check_spinlock_acquired(cachep); 2892 2893 /* 2894 * The slab was either on partial or free list so 2895 * there must be at least one object available for 2896 * allocation. 2897 */ 2898 BUG_ON(page->active >= cachep->num); 2899 2900 while (page->active < cachep->num && batchcount--) { 2901 STATS_INC_ALLOCED(cachep); 2902 STATS_INC_ACTIVE(cachep); 2903 STATS_SET_HIGH(cachep); 2904 2905 ac_put_obj(cachep, ac, slab_get_obj(cachep, page, 2906 node)); 2907 } 2908 2909 /* move slabp to correct slabp list: */ 2910 list_del(&page->lru); 2911 if (page->active == cachep->num) 2912 list_add(&page->lru, &n->slabs_full); 2913 else 2914 list_add(&page->lru, &n->slabs_partial); 2915 } 2916 2917 must_grow: 2918 n->free_objects -= ac->avail; 2919 alloc_done: 2920 spin_unlock(&n->list_lock); 2921 2922 if (unlikely(!ac->avail)) { 2923 int x; 2924 force_grow: 2925 x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL); 2926 2927 /* cache_grow can reenable interrupts, then ac could change. */ 2928 ac = cpu_cache_get(cachep); 2929 node = numa_mem_id(); 2930 2931 /* no objects in sight? abort */ 2932 if (!x && (ac->avail == 0 || force_refill)) 2933 return NULL; 2934 2935 if (!ac->avail) /* objects refilled by interrupt? */ 2936 goto retry; 2937 } 2938 ac->touched = 1; 2939 2940 return ac_get_obj(cachep, ac, flags, force_refill); 2941 } 2942 2943 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep, 2944 gfp_t flags) 2945 { 2946 might_sleep_if(flags & __GFP_WAIT); 2947 #if DEBUG 2948 kmem_flagcheck(cachep, flags); 2949 #endif 2950 } 2951 2952 #if DEBUG 2953 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, 2954 gfp_t flags, void *objp, unsigned long caller) 2955 { 2956 if (!objp) 2957 return objp; 2958 if (cachep->flags & SLAB_POISON) { 2959 #ifdef CONFIG_DEBUG_PAGEALLOC 2960 if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) 2961 kernel_map_pages(virt_to_page(objp), 2962 cachep->size / PAGE_SIZE, 1); 2963 else 2964 check_poison_obj(cachep, objp); 2965 #else 2966 check_poison_obj(cachep, objp); 2967 #endif 2968 poison_obj(cachep, objp, POISON_INUSE); 2969 } 2970 if (cachep->flags & SLAB_STORE_USER) 2971 *dbg_userword(cachep, objp) = (void *)caller; 2972 2973 if (cachep->flags & SLAB_RED_ZONE) { 2974 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || 2975 *dbg_redzone2(cachep, objp) != RED_INACTIVE) { 2976 slab_error(cachep, "double free, or memory outside" 2977 " object was overwritten"); 2978 printk(KERN_ERR 2979 "%p: redzone 1:0x%llx, redzone 2:0x%llx\n", 2980 objp, *dbg_redzone1(cachep, objp), 2981 *dbg_redzone2(cachep, objp)); 2982 } 2983 *dbg_redzone1(cachep, objp) = RED_ACTIVE; 2984 *dbg_redzone2(cachep, objp) = RED_ACTIVE; 2985 } 2986 objp += obj_offset(cachep); 2987 if (cachep->ctor && cachep->flags & SLAB_POISON) 2988 cachep->ctor(objp); 2989 if (ARCH_SLAB_MINALIGN && 2990 ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) { 2991 printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n", 2992 objp, (int)ARCH_SLAB_MINALIGN); 2993 } 2994 return objp; 2995 } 2996 #else 2997 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp) 2998 #endif 2999 3000 static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags) 3001 { 3002 if (cachep == kmem_cache) 3003 return false; 3004 3005 return should_failslab(cachep->object_size, flags, cachep->flags); 3006 } 3007 3008 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3009 { 3010 void *objp; 3011 struct array_cache *ac; 3012 bool force_refill = false; 3013 3014 check_irq_off(); 3015 3016 ac = cpu_cache_get(cachep); 3017 if (likely(ac->avail)) { 3018 ac->touched = 1; 3019 objp = ac_get_obj(cachep, ac, flags, false); 3020 3021 /* 3022 * Allow for the possibility all avail objects are not allowed 3023 * by the current flags 3024 */ 3025 if (objp) { 3026 STATS_INC_ALLOCHIT(cachep); 3027 goto out; 3028 } 3029 force_refill = true; 3030 } 3031 3032 STATS_INC_ALLOCMISS(cachep); 3033 objp = cache_alloc_refill(cachep, flags, force_refill); 3034 /* 3035 * the 'ac' may be updated by cache_alloc_refill(), 3036 * and kmemleak_erase() requires its correct value. 3037 */ 3038 ac = cpu_cache_get(cachep); 3039 3040 out: 3041 /* 3042 * To avoid a false negative, if an object that is in one of the 3043 * per-CPU caches is leaked, we need to make sure kmemleak doesn't 3044 * treat the array pointers as a reference to the object. 3045 */ 3046 if (objp) 3047 kmemleak_erase(&ac->entry[ac->avail]); 3048 return objp; 3049 } 3050 3051 #ifdef CONFIG_NUMA 3052 /* 3053 * Try allocating on another node if PF_SPREAD_SLAB is a mempolicy is set. 3054 * 3055 * If we are in_interrupt, then process context, including cpusets and 3056 * mempolicy, may not apply and should not be used for allocation policy. 3057 */ 3058 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags) 3059 { 3060 int nid_alloc, nid_here; 3061 3062 if (in_interrupt() || (flags & __GFP_THISNODE)) 3063 return NULL; 3064 nid_alloc = nid_here = numa_mem_id(); 3065 if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD)) 3066 nid_alloc = cpuset_slab_spread_node(); 3067 else if (current->mempolicy) 3068 nid_alloc = mempolicy_slab_node(); 3069 if (nid_alloc != nid_here) 3070 return ____cache_alloc_node(cachep, flags, nid_alloc); 3071 return NULL; 3072 } 3073 3074 /* 3075 * Fallback function if there was no memory available and no objects on a 3076 * certain node and fall back is permitted. First we scan all the 3077 * available node for available objects. If that fails then we 3078 * perform an allocation without specifying a node. This allows the page 3079 * allocator to do its reclaim / fallback magic. We then insert the 3080 * slab into the proper nodelist and then allocate from it. 3081 */ 3082 static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags) 3083 { 3084 struct zonelist *zonelist; 3085 gfp_t local_flags; 3086 struct zoneref *z; 3087 struct zone *zone; 3088 enum zone_type high_zoneidx = gfp_zone(flags); 3089 void *obj = NULL; 3090 int nid; 3091 unsigned int cpuset_mems_cookie; 3092 3093 if (flags & __GFP_THISNODE) 3094 return NULL; 3095 3096 local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK); 3097 3098 retry_cpuset: 3099 cpuset_mems_cookie = read_mems_allowed_begin(); 3100 zonelist = node_zonelist(mempolicy_slab_node(), flags); 3101 3102 retry: 3103 /* 3104 * Look through allowed nodes for objects available 3105 * from existing per node queues. 3106 */ 3107 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 3108 nid = zone_to_nid(zone); 3109 3110 if (cpuset_zone_allowed_hardwall(zone, flags) && 3111 cache->node[nid] && 3112 cache->node[nid]->free_objects) { 3113 obj = ____cache_alloc_node(cache, 3114 flags | GFP_THISNODE, nid); 3115 if (obj) 3116 break; 3117 } 3118 } 3119 3120 if (!obj) { 3121 /* 3122 * This allocation will be performed within the constraints 3123 * of the current cpuset / memory policy requirements. 3124 * We may trigger various forms of reclaim on the allowed 3125 * set and go into memory reserves if necessary. 3126 */ 3127 struct page *page; 3128 3129 if (local_flags & __GFP_WAIT) 3130 local_irq_enable(); 3131 kmem_flagcheck(cache, flags); 3132 page = kmem_getpages(cache, local_flags, numa_mem_id()); 3133 if (local_flags & __GFP_WAIT) 3134 local_irq_disable(); 3135 if (page) { 3136 /* 3137 * Insert into the appropriate per node queues 3138 */ 3139 nid = page_to_nid(page); 3140 if (cache_grow(cache, flags, nid, page)) { 3141 obj = ____cache_alloc_node(cache, 3142 flags | GFP_THISNODE, nid); 3143 if (!obj) 3144 /* 3145 * Another processor may allocate the 3146 * objects in the slab since we are 3147 * not holding any locks. 3148 */ 3149 goto retry; 3150 } else { 3151 /* cache_grow already freed obj */ 3152 obj = NULL; 3153 } 3154 } 3155 } 3156 3157 if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie))) 3158 goto retry_cpuset; 3159 return obj; 3160 } 3161 3162 /* 3163 * A interface to enable slab creation on nodeid 3164 */ 3165 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, 3166 int nodeid) 3167 { 3168 struct list_head *entry; 3169 struct page *page; 3170 struct kmem_cache_node *n; 3171 void *obj; 3172 int x; 3173 3174 VM_BUG_ON(nodeid > num_online_nodes()); 3175 n = cachep->node[nodeid]; 3176 BUG_ON(!n); 3177 3178 retry: 3179 check_irq_off(); 3180 spin_lock(&n->list_lock); 3181 entry = n->slabs_partial.next; 3182 if (entry == &n->slabs_partial) { 3183 n->free_touched = 1; 3184 entry = n->slabs_free.next; 3185 if (entry == &n->slabs_free) 3186 goto must_grow; 3187 } 3188 3189 page = list_entry(entry, struct page, lru); 3190 check_spinlock_acquired_node(cachep, nodeid); 3191 3192 STATS_INC_NODEALLOCS(cachep); 3193 STATS_INC_ACTIVE(cachep); 3194 STATS_SET_HIGH(cachep); 3195 3196 BUG_ON(page->active == cachep->num); 3197 3198 obj = slab_get_obj(cachep, page, nodeid); 3199 n->free_objects--; 3200 /* move slabp to correct slabp list: */ 3201 list_del(&page->lru); 3202 3203 if (page->active == cachep->num) 3204 list_add(&page->lru, &n->slabs_full); 3205 else 3206 list_add(&page->lru, &n->slabs_partial); 3207 3208 spin_unlock(&n->list_lock); 3209 goto done; 3210 3211 must_grow: 3212 spin_unlock(&n->list_lock); 3213 x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL); 3214 if (x) 3215 goto retry; 3216 3217 return fallback_alloc(cachep, flags); 3218 3219 done: 3220 return obj; 3221 } 3222 3223 static __always_inline void * 3224 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, 3225 unsigned long caller) 3226 { 3227 unsigned long save_flags; 3228 void *ptr; 3229 int slab_node = numa_mem_id(); 3230 3231 flags &= gfp_allowed_mask; 3232 3233 lockdep_trace_alloc(flags); 3234 3235 if (slab_should_failslab(cachep, flags)) 3236 return NULL; 3237 3238 cachep = memcg_kmem_get_cache(cachep, flags); 3239 3240 cache_alloc_debugcheck_before(cachep, flags); 3241 local_irq_save(save_flags); 3242 3243 if (nodeid == NUMA_NO_NODE) 3244 nodeid = slab_node; 3245 3246 if (unlikely(!cachep->node[nodeid])) { 3247 /* Node not bootstrapped yet */ 3248 ptr = fallback_alloc(cachep, flags); 3249 goto out; 3250 } 3251 3252 if (nodeid == slab_node) { 3253 /* 3254 * Use the locally cached objects if possible. 3255 * However ____cache_alloc does not allow fallback 3256 * to other nodes. It may fail while we still have 3257 * objects on other nodes available. 3258 */ 3259 ptr = ____cache_alloc(cachep, flags); 3260 if (ptr) 3261 goto out; 3262 } 3263 /* ___cache_alloc_node can fall back to other nodes */ 3264 ptr = ____cache_alloc_node(cachep, flags, nodeid); 3265 out: 3266 local_irq_restore(save_flags); 3267 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller); 3268 kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags, 3269 flags); 3270 3271 if (likely(ptr)) { 3272 kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size); 3273 if (unlikely(flags & __GFP_ZERO)) 3274 memset(ptr, 0, cachep->object_size); 3275 } 3276 3277 return ptr; 3278 } 3279 3280 static __always_inline void * 3281 __do_cache_alloc(struct kmem_cache *cache, gfp_t flags) 3282 { 3283 void *objp; 3284 3285 if (current->mempolicy || unlikely(current->flags & PF_SPREAD_SLAB)) { 3286 objp = alternate_node_alloc(cache, flags); 3287 if (objp) 3288 goto out; 3289 } 3290 objp = ____cache_alloc(cache, flags); 3291 3292 /* 3293 * We may just have run out of memory on the local node. 3294 * ____cache_alloc_node() knows how to locate memory on other nodes 3295 */ 3296 if (!objp) 3297 objp = ____cache_alloc_node(cache, flags, numa_mem_id()); 3298 3299 out: 3300 return objp; 3301 } 3302 #else 3303 3304 static __always_inline void * 3305 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3306 { 3307 return ____cache_alloc(cachep, flags); 3308 } 3309 3310 #endif /* CONFIG_NUMA */ 3311 3312 static __always_inline void * 3313 slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller) 3314 { 3315 unsigned long save_flags; 3316 void *objp; 3317 3318 flags &= gfp_allowed_mask; 3319 3320 lockdep_trace_alloc(flags); 3321 3322 if (slab_should_failslab(cachep, flags)) 3323 return NULL; 3324 3325 cachep = memcg_kmem_get_cache(cachep, flags); 3326 3327 cache_alloc_debugcheck_before(cachep, flags); 3328 local_irq_save(save_flags); 3329 objp = __do_cache_alloc(cachep, flags); 3330 local_irq_restore(save_flags); 3331 objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller); 3332 kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags, 3333 flags); 3334 prefetchw(objp); 3335 3336 if (likely(objp)) { 3337 kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size); 3338 if (unlikely(flags & __GFP_ZERO)) 3339 memset(objp, 0, cachep->object_size); 3340 } 3341 3342 return objp; 3343 } 3344 3345 /* 3346 * Caller needs to acquire correct kmem_cache_node's list_lock 3347 */ 3348 static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects, 3349 int node) 3350 { 3351 int i; 3352 struct kmem_cache_node *n; 3353 3354 for (i = 0; i < nr_objects; i++) { 3355 void *objp; 3356 struct page *page; 3357 3358 clear_obj_pfmemalloc(&objpp[i]); 3359 objp = objpp[i]; 3360 3361 page = virt_to_head_page(objp); 3362 n = cachep->node[node]; 3363 list_del(&page->lru); 3364 check_spinlock_acquired_node(cachep, node); 3365 slab_put_obj(cachep, page, objp, node); 3366 STATS_DEC_ACTIVE(cachep); 3367 n->free_objects++; 3368 3369 /* fixup slab chains */ 3370 if (page->active == 0) { 3371 if (n->free_objects > n->free_limit) { 3372 n->free_objects -= cachep->num; 3373 /* No need to drop any previously held 3374 * lock here, even if we have a off-slab slab 3375 * descriptor it is guaranteed to come from 3376 * a different cache, refer to comments before 3377 * alloc_slabmgmt. 3378 */ 3379 slab_destroy(cachep, page); 3380 } else { 3381 list_add(&page->lru, &n->slabs_free); 3382 } 3383 } else { 3384 /* Unconditionally move a slab to the end of the 3385 * partial list on free - maximum time for the 3386 * other objects to be freed, too. 3387 */ 3388 list_add_tail(&page->lru, &n->slabs_partial); 3389 } 3390 } 3391 } 3392 3393 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac) 3394 { 3395 int batchcount; 3396 struct kmem_cache_node *n; 3397 int node = numa_mem_id(); 3398 3399 batchcount = ac->batchcount; 3400 #if DEBUG 3401 BUG_ON(!batchcount || batchcount > ac->avail); 3402 #endif 3403 check_irq_off(); 3404 n = cachep->node[node]; 3405 spin_lock(&n->list_lock); 3406 if (n->shared) { 3407 struct array_cache *shared_array = n->shared; 3408 int max = shared_array->limit - shared_array->avail; 3409 if (max) { 3410 if (batchcount > max) 3411 batchcount = max; 3412 memcpy(&(shared_array->entry[shared_array->avail]), 3413 ac->entry, sizeof(void *) * batchcount); 3414 shared_array->avail += batchcount; 3415 goto free_done; 3416 } 3417 } 3418 3419 free_block(cachep, ac->entry, batchcount, node); 3420 free_done: 3421 #if STATS 3422 { 3423 int i = 0; 3424 struct list_head *p; 3425 3426 p = n->slabs_free.next; 3427 while (p != &(n->slabs_free)) { 3428 struct page *page; 3429 3430 page = list_entry(p, struct page, lru); 3431 BUG_ON(page->active); 3432 3433 i++; 3434 p = p->next; 3435 } 3436 STATS_SET_FREEABLE(cachep, i); 3437 } 3438 #endif 3439 spin_unlock(&n->list_lock); 3440 ac->avail -= batchcount; 3441 memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail); 3442 } 3443 3444 /* 3445 * Release an obj back to its cache. If the obj has a constructed state, it must 3446 * be in this state _before_ it is released. Called with disabled ints. 3447 */ 3448 static inline void __cache_free(struct kmem_cache *cachep, void *objp, 3449 unsigned long caller) 3450 { 3451 struct array_cache *ac = cpu_cache_get(cachep); 3452 3453 check_irq_off(); 3454 kmemleak_free_recursive(objp, cachep->flags); 3455 objp = cache_free_debugcheck(cachep, objp, caller); 3456 3457 kmemcheck_slab_free(cachep, objp, cachep->object_size); 3458 3459 /* 3460 * Skip calling cache_free_alien() when the platform is not numa. 3461 * This will avoid cache misses that happen while accessing slabp (which 3462 * is per page memory reference) to get nodeid. Instead use a global 3463 * variable to skip the call, which is mostly likely to be present in 3464 * the cache. 3465 */ 3466 if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) 3467 return; 3468 3469 if (likely(ac->avail < ac->limit)) { 3470 STATS_INC_FREEHIT(cachep); 3471 } else { 3472 STATS_INC_FREEMISS(cachep); 3473 cache_flusharray(cachep, ac); 3474 } 3475 3476 ac_put_obj(cachep, ac, objp); 3477 } 3478 3479 /** 3480 * kmem_cache_alloc - Allocate an object 3481 * @cachep: The cache to allocate from. 3482 * @flags: See kmalloc(). 3483 * 3484 * Allocate an object from this cache. The flags are only relevant 3485 * if the cache has no available objects. 3486 */ 3487 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3488 { 3489 void *ret = slab_alloc(cachep, flags, _RET_IP_); 3490 3491 trace_kmem_cache_alloc(_RET_IP_, ret, 3492 cachep->object_size, cachep->size, flags); 3493 3494 return ret; 3495 } 3496 EXPORT_SYMBOL(kmem_cache_alloc); 3497 3498 #ifdef CONFIG_TRACING 3499 void * 3500 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size) 3501 { 3502 void *ret; 3503 3504 ret = slab_alloc(cachep, flags, _RET_IP_); 3505 3506 trace_kmalloc(_RET_IP_, ret, 3507 size, cachep->size, flags); 3508 return ret; 3509 } 3510 EXPORT_SYMBOL(kmem_cache_alloc_trace); 3511 #endif 3512 3513 #ifdef CONFIG_NUMA 3514 /** 3515 * kmem_cache_alloc_node - Allocate an object on the specified node 3516 * @cachep: The cache to allocate from. 3517 * @flags: See kmalloc(). 3518 * @nodeid: node number of the target node. 3519 * 3520 * Identical to kmem_cache_alloc but it will allocate memory on the given 3521 * node, which can improve the performance for cpu bound structures. 3522 * 3523 * Fallback to other node is possible if __GFP_THISNODE is not set. 3524 */ 3525 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid) 3526 { 3527 void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); 3528 3529 trace_kmem_cache_alloc_node(_RET_IP_, ret, 3530 cachep->object_size, cachep->size, 3531 flags, nodeid); 3532 3533 return ret; 3534 } 3535 EXPORT_SYMBOL(kmem_cache_alloc_node); 3536 3537 #ifdef CONFIG_TRACING 3538 void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep, 3539 gfp_t flags, 3540 int nodeid, 3541 size_t size) 3542 { 3543 void *ret; 3544 3545 ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_); 3546 3547 trace_kmalloc_node(_RET_IP_, ret, 3548 size, cachep->size, 3549 flags, nodeid); 3550 return ret; 3551 } 3552 EXPORT_SYMBOL(kmem_cache_alloc_node_trace); 3553 #endif 3554 3555 static __always_inline void * 3556 __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller) 3557 { 3558 struct kmem_cache *cachep; 3559 3560 cachep = kmalloc_slab(size, flags); 3561 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3562 return cachep; 3563 return kmem_cache_alloc_node_trace(cachep, flags, node, size); 3564 } 3565 3566 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) 3567 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3568 { 3569 return __do_kmalloc_node(size, flags, node, _RET_IP_); 3570 } 3571 EXPORT_SYMBOL(__kmalloc_node); 3572 3573 void *__kmalloc_node_track_caller(size_t size, gfp_t flags, 3574 int node, unsigned long caller) 3575 { 3576 return __do_kmalloc_node(size, flags, node, caller); 3577 } 3578 EXPORT_SYMBOL(__kmalloc_node_track_caller); 3579 #else 3580 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3581 { 3582 return __do_kmalloc_node(size, flags, node, 0); 3583 } 3584 EXPORT_SYMBOL(__kmalloc_node); 3585 #endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */ 3586 #endif /* CONFIG_NUMA */ 3587 3588 /** 3589 * __do_kmalloc - allocate memory 3590 * @size: how many bytes of memory are required. 3591 * @flags: the type of memory to allocate (see kmalloc). 3592 * @caller: function caller for debug tracking of the caller 3593 */ 3594 static __always_inline void *__do_kmalloc(size_t size, gfp_t flags, 3595 unsigned long caller) 3596 { 3597 struct kmem_cache *cachep; 3598 void *ret; 3599 3600 cachep = kmalloc_slab(size, flags); 3601 if (unlikely(ZERO_OR_NULL_PTR(cachep))) 3602 return cachep; 3603 ret = slab_alloc(cachep, flags, caller); 3604 3605 trace_kmalloc(caller, ret, 3606 size, cachep->size, flags); 3607 3608 return ret; 3609 } 3610 3611 3612 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING) 3613 void *__kmalloc(size_t size, gfp_t flags) 3614 { 3615 return __do_kmalloc(size, flags, _RET_IP_); 3616 } 3617 EXPORT_SYMBOL(__kmalloc); 3618 3619 void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller) 3620 { 3621 return __do_kmalloc(size, flags, caller); 3622 } 3623 EXPORT_SYMBOL(__kmalloc_track_caller); 3624 3625 #else 3626 void *__kmalloc(size_t size, gfp_t flags) 3627 { 3628 return __do_kmalloc(size, flags, 0); 3629 } 3630 EXPORT_SYMBOL(__kmalloc); 3631 #endif 3632 3633 /** 3634 * kmem_cache_free - Deallocate an object 3635 * @cachep: The cache the allocation was from. 3636 * @objp: The previously allocated object. 3637 * 3638 * Free an object which was previously allocated from this 3639 * cache. 3640 */ 3641 void kmem_cache_free(struct kmem_cache *cachep, void *objp) 3642 { 3643 unsigned long flags; 3644 cachep = cache_from_obj(cachep, objp); 3645 if (!cachep) 3646 return; 3647 3648 local_irq_save(flags); 3649 debug_check_no_locks_freed(objp, cachep->object_size); 3650 if (!(cachep->flags & SLAB_DEBUG_OBJECTS)) 3651 debug_check_no_obj_freed(objp, cachep->object_size); 3652 __cache_free(cachep, objp, _RET_IP_); 3653 local_irq_restore(flags); 3654 3655 trace_kmem_cache_free(_RET_IP_, objp); 3656 } 3657 EXPORT_SYMBOL(kmem_cache_free); 3658 3659 /** 3660 * kfree - free previously allocated memory 3661 * @objp: pointer returned by kmalloc. 3662 * 3663 * If @objp is NULL, no operation is performed. 3664 * 3665 * Don't free memory not originally allocated by kmalloc() 3666 * or you will run into trouble. 3667 */ 3668 void kfree(const void *objp) 3669 { 3670 struct kmem_cache *c; 3671 unsigned long flags; 3672 3673 trace_kfree(_RET_IP_, objp); 3674 3675 if (unlikely(ZERO_OR_NULL_PTR(objp))) 3676 return; 3677 local_irq_save(flags); 3678 kfree_debugcheck(objp); 3679 c = virt_to_cache(objp); 3680 debug_check_no_locks_freed(objp, c->object_size); 3681 3682 debug_check_no_obj_freed(objp, c->object_size); 3683 __cache_free(c, (void *)objp, _RET_IP_); 3684 local_irq_restore(flags); 3685 } 3686 EXPORT_SYMBOL(kfree); 3687 3688 /* 3689 * This initializes kmem_cache_node or resizes various caches for all nodes. 3690 */ 3691 static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp) 3692 { 3693 int node; 3694 struct kmem_cache_node *n; 3695 struct array_cache *new_shared; 3696 struct array_cache **new_alien = NULL; 3697 3698 for_each_online_node(node) { 3699 3700 if (use_alien_caches) { 3701 new_alien = alloc_alien_cache(node, cachep->limit, gfp); 3702 if (!new_alien) 3703 goto fail; 3704 } 3705 3706 new_shared = NULL; 3707 if (cachep->shared) { 3708 new_shared = alloc_arraycache(node, 3709 cachep->shared*cachep->batchcount, 3710 0xbaadf00d, gfp); 3711 if (!new_shared) { 3712 free_alien_cache(new_alien); 3713 goto fail; 3714 } 3715 } 3716 3717 n = cachep->node[node]; 3718 if (n) { 3719 struct array_cache *shared = n->shared; 3720 3721 spin_lock_irq(&n->list_lock); 3722 3723 if (shared) 3724 free_block(cachep, shared->entry, 3725 shared->avail, node); 3726 3727 n->shared = new_shared; 3728 if (!n->alien) { 3729 n->alien = new_alien; 3730 new_alien = NULL; 3731 } 3732 n->free_limit = (1 + nr_cpus_node(node)) * 3733 cachep->batchcount + cachep->num; 3734 spin_unlock_irq(&n->list_lock); 3735 kfree(shared); 3736 free_alien_cache(new_alien); 3737 continue; 3738 } 3739 n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node); 3740 if (!n) { 3741 free_alien_cache(new_alien); 3742 kfree(new_shared); 3743 goto fail; 3744 } 3745 3746 kmem_cache_node_init(n); 3747 n->next_reap = jiffies + REAPTIMEOUT_NODE + 3748 ((unsigned long)cachep) % REAPTIMEOUT_NODE; 3749 n->shared = new_shared; 3750 n->alien = new_alien; 3751 n->free_limit = (1 + nr_cpus_node(node)) * 3752 cachep->batchcount + cachep->num; 3753 cachep->node[node] = n; 3754 } 3755 return 0; 3756 3757 fail: 3758 if (!cachep->list.next) { 3759 /* Cache is not active yet. Roll back what we did */ 3760 node--; 3761 while (node >= 0) { 3762 if (cachep->node[node]) { 3763 n = cachep->node[node]; 3764 3765 kfree(n->shared); 3766 free_alien_cache(n->alien); 3767 kfree(n); 3768 cachep->node[node] = NULL; 3769 } 3770 node--; 3771 } 3772 } 3773 return -ENOMEM; 3774 } 3775 3776 struct ccupdate_struct { 3777 struct kmem_cache *cachep; 3778 struct array_cache *new[0]; 3779 }; 3780 3781 static void do_ccupdate_local(void *info) 3782 { 3783 struct ccupdate_struct *new = info; 3784 struct array_cache *old; 3785 3786 check_irq_off(); 3787 old = cpu_cache_get(new->cachep); 3788 3789 new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()]; 3790 new->new[smp_processor_id()] = old; 3791 } 3792 3793 /* Always called with the slab_mutex held */ 3794 static int __do_tune_cpucache(struct kmem_cache *cachep, int limit, 3795 int batchcount, int shared, gfp_t gfp) 3796 { 3797 struct ccupdate_struct *new; 3798 int i; 3799 3800 new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *), 3801 gfp); 3802 if (!new) 3803 return -ENOMEM; 3804 3805 for_each_online_cpu(i) { 3806 new->new[i] = alloc_arraycache(cpu_to_mem(i), limit, 3807 batchcount, gfp); 3808 if (!new->new[i]) { 3809 for (i--; i >= 0; i--) 3810 kfree(new->new[i]); 3811 kfree(new); 3812 return -ENOMEM; 3813 } 3814 } 3815 new->cachep = cachep; 3816 3817 on_each_cpu(do_ccupdate_local, (void *)new, 1); 3818 3819 check_irq_on(); 3820 cachep->batchcount = batchcount; 3821 cachep->limit = limit; 3822 cachep->shared = shared; 3823 3824 for_each_online_cpu(i) { 3825 struct array_cache *ccold = new->new[i]; 3826 if (!ccold) 3827 continue; 3828 spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); 3829 free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i)); 3830 spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock); 3831 kfree(ccold); 3832 } 3833 kfree(new); 3834 return alloc_kmem_cache_node(cachep, gfp); 3835 } 3836 3837 static int do_tune_cpucache(struct kmem_cache *cachep, int limit, 3838 int batchcount, int shared, gfp_t gfp) 3839 { 3840 int ret; 3841 struct kmem_cache *c = NULL; 3842 int i = 0; 3843 3844 ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp); 3845 3846 if (slab_state < FULL) 3847 return ret; 3848 3849 if ((ret < 0) || !is_root_cache(cachep)) 3850 return ret; 3851 3852 VM_BUG_ON(!mutex_is_locked(&slab_mutex)); 3853 for_each_memcg_cache_index(i) { 3854 c = cache_from_memcg_idx(cachep, i); 3855 if (c) 3856 /* return value determined by the parent cache only */ 3857 __do_tune_cpucache(c, limit, batchcount, shared, gfp); 3858 } 3859 3860 return ret; 3861 } 3862 3863 /* Called with slab_mutex held always */ 3864 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp) 3865 { 3866 int err; 3867 int limit = 0; 3868 int shared = 0; 3869 int batchcount = 0; 3870 3871 if (!is_root_cache(cachep)) { 3872 struct kmem_cache *root = memcg_root_cache(cachep); 3873 limit = root->limit; 3874 shared = root->shared; 3875 batchcount = root->batchcount; 3876 } 3877 3878 if (limit && shared && batchcount) 3879 goto skip_setup; 3880 /* 3881 * The head array serves three purposes: 3882 * - create a LIFO ordering, i.e. return objects that are cache-warm 3883 * - reduce the number of spinlock operations. 3884 * - reduce the number of linked list operations on the slab and 3885 * bufctl chains: array operations are cheaper. 3886 * The numbers are guessed, we should auto-tune as described by 3887 * Bonwick. 3888 */ 3889 if (cachep->size > 131072) 3890 limit = 1; 3891 else if (cachep->size > PAGE_SIZE) 3892 limit = 8; 3893 else if (cachep->size > 1024) 3894 limit = 24; 3895 else if (cachep->size > 256) 3896 limit = 54; 3897 else 3898 limit = 120; 3899 3900 /* 3901 * CPU bound tasks (e.g. network routing) can exhibit cpu bound 3902 * allocation behaviour: Most allocs on one cpu, most free operations 3903 * on another cpu. For these cases, an efficient object passing between 3904 * cpus is necessary. This is provided by a shared array. The array 3905 * replaces Bonwick's magazine layer. 3906 * On uniprocessor, it's functionally equivalent (but less efficient) 3907 * to a larger limit. Thus disabled by default. 3908 */ 3909 shared = 0; 3910 if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1) 3911 shared = 8; 3912 3913 #if DEBUG 3914 /* 3915 * With debugging enabled, large batchcount lead to excessively long 3916 * periods with disabled local interrupts. Limit the batchcount 3917 */ 3918 if (limit > 32) 3919 limit = 32; 3920 #endif 3921 batchcount = (limit + 1) / 2; 3922 skip_setup: 3923 err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp); 3924 if (err) 3925 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", 3926 cachep->name, -err); 3927 return err; 3928 } 3929 3930 /* 3931 * Drain an array if it contains any elements taking the node lock only if 3932 * necessary. Note that the node listlock also protects the array_cache 3933 * if drain_array() is used on the shared array. 3934 */ 3935 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n, 3936 struct array_cache *ac, int force, int node) 3937 { 3938 int tofree; 3939 3940 if (!ac || !ac->avail) 3941 return; 3942 if (ac->touched && !force) { 3943 ac->touched = 0; 3944 } else { 3945 spin_lock_irq(&n->list_lock); 3946 if (ac->avail) { 3947 tofree = force ? ac->avail : (ac->limit + 4) / 5; 3948 if (tofree > ac->avail) 3949 tofree = (ac->avail + 1) / 2; 3950 free_block(cachep, ac->entry, tofree, node); 3951 ac->avail -= tofree; 3952 memmove(ac->entry, &(ac->entry[tofree]), 3953 sizeof(void *) * ac->avail); 3954 } 3955 spin_unlock_irq(&n->list_lock); 3956 } 3957 } 3958 3959 /** 3960 * cache_reap - Reclaim memory from caches. 3961 * @w: work descriptor 3962 * 3963 * Called from workqueue/eventd every few seconds. 3964 * Purpose: 3965 * - clear the per-cpu caches for this CPU. 3966 * - return freeable pages to the main free memory pool. 3967 * 3968 * If we cannot acquire the cache chain mutex then just give up - we'll try 3969 * again on the next iteration. 3970 */ 3971 static void cache_reap(struct work_struct *w) 3972 { 3973 struct kmem_cache *searchp; 3974 struct kmem_cache_node *n; 3975 int node = numa_mem_id(); 3976 struct delayed_work *work = to_delayed_work(w); 3977 3978 if (!mutex_trylock(&slab_mutex)) 3979 /* Give up. Setup the next iteration. */ 3980 goto out; 3981 3982 list_for_each_entry(searchp, &slab_caches, list) { 3983 check_irq_on(); 3984 3985 /* 3986 * We only take the node lock if absolutely necessary and we 3987 * have established with reasonable certainty that 3988 * we can do some work if the lock was obtained. 3989 */ 3990 n = searchp->node[node]; 3991 3992 reap_alien(searchp, n); 3993 3994 drain_array(searchp, n, cpu_cache_get(searchp), 0, node); 3995 3996 /* 3997 * These are racy checks but it does not matter 3998 * if we skip one check or scan twice. 3999 */ 4000 if (time_after(n->next_reap, jiffies)) 4001 goto next; 4002 4003 n->next_reap = jiffies + REAPTIMEOUT_NODE; 4004 4005 drain_array(searchp, n, n->shared, 0, node); 4006 4007 if (n->free_touched) 4008 n->free_touched = 0; 4009 else { 4010 int freed; 4011 4012 freed = drain_freelist(searchp, n, (n->free_limit + 4013 5 * searchp->num - 1) / (5 * searchp->num)); 4014 STATS_ADD_REAPED(searchp, freed); 4015 } 4016 next: 4017 cond_resched(); 4018 } 4019 check_irq_on(); 4020 mutex_unlock(&slab_mutex); 4021 next_reap_node(); 4022 out: 4023 /* Set up the next iteration */ 4024 schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC)); 4025 } 4026 4027 #ifdef CONFIG_SLABINFO 4028 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo) 4029 { 4030 struct page *page; 4031 unsigned long active_objs; 4032 unsigned long num_objs; 4033 unsigned long active_slabs = 0; 4034 unsigned long num_slabs, free_objects = 0, shared_avail = 0; 4035 const char *name; 4036 char *error = NULL; 4037 int node; 4038 struct kmem_cache_node *n; 4039 4040 active_objs = 0; 4041 num_slabs = 0; 4042 for_each_online_node(node) { 4043 n = cachep->node[node]; 4044 if (!n) 4045 continue; 4046 4047 check_irq_on(); 4048 spin_lock_irq(&n->list_lock); 4049 4050 list_for_each_entry(page, &n->slabs_full, lru) { 4051 if (page->active != cachep->num && !error) 4052 error = "slabs_full accounting error"; 4053 active_objs += cachep->num; 4054 active_slabs++; 4055 } 4056 list_for_each_entry(page, &n->slabs_partial, lru) { 4057 if (page->active == cachep->num && !error) 4058 error = "slabs_partial accounting error"; 4059 if (!page->active && !error) 4060 error = "slabs_partial accounting error"; 4061 active_objs += page->active; 4062 active_slabs++; 4063 } 4064 list_for_each_entry(page, &n->slabs_free, lru) { 4065 if (page->active && !error) 4066 error = "slabs_free accounting error"; 4067 num_slabs++; 4068 } 4069 free_objects += n->free_objects; 4070 if (n->shared) 4071 shared_avail += n->shared->avail; 4072 4073 spin_unlock_irq(&n->list_lock); 4074 } 4075 num_slabs += active_slabs; 4076 num_objs = num_slabs * cachep->num; 4077 if (num_objs - active_objs != free_objects && !error) 4078 error = "free_objects accounting error"; 4079 4080 name = cachep->name; 4081 if (error) 4082 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 4083 4084 sinfo->active_objs = active_objs; 4085 sinfo->num_objs = num_objs; 4086 sinfo->active_slabs = active_slabs; 4087 sinfo->num_slabs = num_slabs; 4088 sinfo->shared_avail = shared_avail; 4089 sinfo->limit = cachep->limit; 4090 sinfo->batchcount = cachep->batchcount; 4091 sinfo->shared = cachep->shared; 4092 sinfo->objects_per_slab = cachep->num; 4093 sinfo->cache_order = cachep->gfporder; 4094 } 4095 4096 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep) 4097 { 4098 #if STATS 4099 { /* node stats */ 4100 unsigned long high = cachep->high_mark; 4101 unsigned long allocs = cachep->num_allocations; 4102 unsigned long grown = cachep->grown; 4103 unsigned long reaped = cachep->reaped; 4104 unsigned long errors = cachep->errors; 4105 unsigned long max_freeable = cachep->max_freeable; 4106 unsigned long node_allocs = cachep->node_allocs; 4107 unsigned long node_frees = cachep->node_frees; 4108 unsigned long overflows = cachep->node_overflow; 4109 4110 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu " 4111 "%4lu %4lu %4lu %4lu %4lu", 4112 allocs, high, grown, 4113 reaped, errors, max_freeable, node_allocs, 4114 node_frees, overflows); 4115 } 4116 /* cpu stats */ 4117 { 4118 unsigned long allochit = atomic_read(&cachep->allochit); 4119 unsigned long allocmiss = atomic_read(&cachep->allocmiss); 4120 unsigned long freehit = atomic_read(&cachep->freehit); 4121 unsigned long freemiss = atomic_read(&cachep->freemiss); 4122 4123 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", 4124 allochit, allocmiss, freehit, freemiss); 4125 } 4126 #endif 4127 } 4128 4129 #define MAX_SLABINFO_WRITE 128 4130 /** 4131 * slabinfo_write - Tuning for the slab allocator 4132 * @file: unused 4133 * @buffer: user buffer 4134 * @count: data length 4135 * @ppos: unused 4136 */ 4137 ssize_t slabinfo_write(struct file *file, const char __user *buffer, 4138 size_t count, loff_t *ppos) 4139 { 4140 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; 4141 int limit, batchcount, shared, res; 4142 struct kmem_cache *cachep; 4143 4144 if (count > MAX_SLABINFO_WRITE) 4145 return -EINVAL; 4146 if (copy_from_user(&kbuf, buffer, count)) 4147 return -EFAULT; 4148 kbuf[MAX_SLABINFO_WRITE] = '\0'; 4149 4150 tmp = strchr(kbuf, ' '); 4151 if (!tmp) 4152 return -EINVAL; 4153 *tmp = '\0'; 4154 tmp++; 4155 if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3) 4156 return -EINVAL; 4157 4158 /* Find the cache in the chain of caches. */ 4159 mutex_lock(&slab_mutex); 4160 res = -EINVAL; 4161 list_for_each_entry(cachep, &slab_caches, list) { 4162 if (!strcmp(cachep->name, kbuf)) { 4163 if (limit < 1 || batchcount < 1 || 4164 batchcount > limit || shared < 0) { 4165 res = 0; 4166 } else { 4167 res = do_tune_cpucache(cachep, limit, 4168 batchcount, shared, 4169 GFP_KERNEL); 4170 } 4171 break; 4172 } 4173 } 4174 mutex_unlock(&slab_mutex); 4175 if (res >= 0) 4176 res = count; 4177 return res; 4178 } 4179 4180 #ifdef CONFIG_DEBUG_SLAB_LEAK 4181 4182 static void *leaks_start(struct seq_file *m, loff_t *pos) 4183 { 4184 mutex_lock(&slab_mutex); 4185 return seq_list_start(&slab_caches, *pos); 4186 } 4187 4188 static inline int add_caller(unsigned long *n, unsigned long v) 4189 { 4190 unsigned long *p; 4191 int l; 4192 if (!v) 4193 return 1; 4194 l = n[1]; 4195 p = n + 2; 4196 while (l) { 4197 int i = l/2; 4198 unsigned long *q = p + 2 * i; 4199 if (*q == v) { 4200 q[1]++; 4201 return 1; 4202 } 4203 if (*q > v) { 4204 l = i; 4205 } else { 4206 p = q + 2; 4207 l -= i + 1; 4208 } 4209 } 4210 if (++n[1] == n[0]) 4211 return 0; 4212 memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n)); 4213 p[0] = v; 4214 p[1] = 1; 4215 return 1; 4216 } 4217 4218 static void handle_slab(unsigned long *n, struct kmem_cache *c, 4219 struct page *page) 4220 { 4221 void *p; 4222 int i, j; 4223 4224 if (n[0] == n[1]) 4225 return; 4226 for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) { 4227 bool active = true; 4228 4229 for (j = page->active; j < c->num; j++) { 4230 /* Skip freed item */ 4231 if (get_free_obj(page, j) == i) { 4232 active = false; 4233 break; 4234 } 4235 } 4236 if (!active) 4237 continue; 4238 4239 if (!add_caller(n, (unsigned long)*dbg_userword(c, p))) 4240 return; 4241 } 4242 } 4243 4244 static void show_symbol(struct seq_file *m, unsigned long address) 4245 { 4246 #ifdef CONFIG_KALLSYMS 4247 unsigned long offset, size; 4248 char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN]; 4249 4250 if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) { 4251 seq_printf(m, "%s+%#lx/%#lx", name, offset, size); 4252 if (modname[0]) 4253 seq_printf(m, " [%s]", modname); 4254 return; 4255 } 4256 #endif 4257 seq_printf(m, "%p", (void *)address); 4258 } 4259 4260 static int leaks_show(struct seq_file *m, void *p) 4261 { 4262 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list); 4263 struct page *page; 4264 struct kmem_cache_node *n; 4265 const char *name; 4266 unsigned long *x = m->private; 4267 int node; 4268 int i; 4269 4270 if (!(cachep->flags & SLAB_STORE_USER)) 4271 return 0; 4272 if (!(cachep->flags & SLAB_RED_ZONE)) 4273 return 0; 4274 4275 /* OK, we can do it */ 4276 4277 x[1] = 0; 4278 4279 for_each_online_node(node) { 4280 n = cachep->node[node]; 4281 if (!n) 4282 continue; 4283 4284 check_irq_on(); 4285 spin_lock_irq(&n->list_lock); 4286 4287 list_for_each_entry(page, &n->slabs_full, lru) 4288 handle_slab(x, cachep, page); 4289 list_for_each_entry(page, &n->slabs_partial, lru) 4290 handle_slab(x, cachep, page); 4291 spin_unlock_irq(&n->list_lock); 4292 } 4293 name = cachep->name; 4294 if (x[0] == x[1]) { 4295 /* Increase the buffer size */ 4296 mutex_unlock(&slab_mutex); 4297 m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL); 4298 if (!m->private) { 4299 /* Too bad, we are really out */ 4300 m->private = x; 4301 mutex_lock(&slab_mutex); 4302 return -ENOMEM; 4303 } 4304 *(unsigned long *)m->private = x[0] * 2; 4305 kfree(x); 4306 mutex_lock(&slab_mutex); 4307 /* Now make sure this entry will be retried */ 4308 m->count = m->size; 4309 return 0; 4310 } 4311 for (i = 0; i < x[1]; i++) { 4312 seq_printf(m, "%s: %lu ", name, x[2*i+3]); 4313 show_symbol(m, x[2*i+2]); 4314 seq_putc(m, '\n'); 4315 } 4316 4317 return 0; 4318 } 4319 4320 static const struct seq_operations slabstats_op = { 4321 .start = leaks_start, 4322 .next = slab_next, 4323 .stop = slab_stop, 4324 .show = leaks_show, 4325 }; 4326 4327 static int slabstats_open(struct inode *inode, struct file *file) 4328 { 4329 unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL); 4330 int ret = -ENOMEM; 4331 if (n) { 4332 ret = seq_open(file, &slabstats_op); 4333 if (!ret) { 4334 struct seq_file *m = file->private_data; 4335 *n = PAGE_SIZE / (2 * sizeof(unsigned long)); 4336 m->private = n; 4337 n = NULL; 4338 } 4339 kfree(n); 4340 } 4341 return ret; 4342 } 4343 4344 static const struct file_operations proc_slabstats_operations = { 4345 .open = slabstats_open, 4346 .read = seq_read, 4347 .llseek = seq_lseek, 4348 .release = seq_release_private, 4349 }; 4350 #endif 4351 4352 static int __init slab_proc_init(void) 4353 { 4354 #ifdef CONFIG_DEBUG_SLAB_LEAK 4355 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations); 4356 #endif 4357 return 0; 4358 } 4359 module_init(slab_proc_init); 4360 #endif 4361 4362 /** 4363 * ksize - get the actual amount of memory allocated for a given object 4364 * @objp: Pointer to the object 4365 * 4366 * kmalloc may internally round up allocations and return more memory 4367 * than requested. ksize() can be used to determine the actual amount of 4368 * memory allocated. The caller may use this additional memory, even though 4369 * a smaller amount of memory was initially specified with the kmalloc call. 4370 * The caller must guarantee that objp points to a valid object previously 4371 * allocated with either kmalloc() or kmem_cache_alloc(). The object 4372 * must not be freed during the duration of the call. 4373 */ 4374 size_t ksize(const void *objp) 4375 { 4376 BUG_ON(!objp); 4377 if (unlikely(objp == ZERO_SIZE_PTR)) 4378 return 0; 4379 4380 return virt_to_cache(objp)->object_size; 4381 } 4382 EXPORT_SYMBOL(ksize); 4383