xref: /openbmc/linux/mm/slab.c (revision 84d517f3)
1 /*
2  * linux/mm/slab.c
3  * Written by Mark Hemment, 1996/97.
4  * (markhe@nextd.demon.co.uk)
5  *
6  * kmem_cache_destroy() + some cleanup - 1999 Andrea Arcangeli
7  *
8  * Major cleanup, different bufctl logic, per-cpu arrays
9  *	(c) 2000 Manfred Spraul
10  *
11  * Cleanup, make the head arrays unconditional, preparation for NUMA
12  * 	(c) 2002 Manfred Spraul
13  *
14  * An implementation of the Slab Allocator as described in outline in;
15  *	UNIX Internals: The New Frontiers by Uresh Vahalia
16  *	Pub: Prentice Hall	ISBN 0-13-101908-2
17  * or with a little more detail in;
18  *	The Slab Allocator: An Object-Caching Kernel Memory Allocator
19  *	Jeff Bonwick (Sun Microsystems).
20  *	Presented at: USENIX Summer 1994 Technical Conference
21  *
22  * The memory is organized in caches, one cache for each object type.
23  * (e.g. inode_cache, dentry_cache, buffer_head, vm_area_struct)
24  * Each cache consists out of many slabs (they are small (usually one
25  * page long) and always contiguous), and each slab contains multiple
26  * initialized objects.
27  *
28  * This means, that your constructor is used only for newly allocated
29  * slabs and you must pass objects with the same initializations to
30  * kmem_cache_free.
31  *
32  * Each cache can only support one memory type (GFP_DMA, GFP_HIGHMEM,
33  * normal). If you need a special memory type, then must create a new
34  * cache for that memory type.
35  *
36  * In order to reduce fragmentation, the slabs are sorted in 3 groups:
37  *   full slabs with 0 free objects
38  *   partial slabs
39  *   empty slabs with no allocated objects
40  *
41  * If partial slabs exist, then new allocations come from these slabs,
42  * otherwise from empty slabs or new slabs are allocated.
43  *
44  * kmem_cache_destroy() CAN CRASH if you try to allocate from the cache
45  * during kmem_cache_destroy(). The caller must prevent concurrent allocs.
46  *
47  * Each cache has a short per-cpu head array, most allocs
48  * and frees go into that array, and if that array overflows, then 1/2
49  * of the entries in the array are given back into the global cache.
50  * The head array is strictly LIFO and should improve the cache hit rates.
51  * On SMP, it additionally reduces the spinlock operations.
52  *
53  * The c_cpuarray may not be read with enabled local interrupts -
54  * it's changed with a smp_call_function().
55  *
56  * SMP synchronization:
57  *  constructors and destructors are called without any locking.
58  *  Several members in struct kmem_cache and struct slab never change, they
59  *	are accessed without any locking.
60  *  The per-cpu arrays are never accessed from the wrong cpu, no locking,
61  *  	and local interrupts are disabled so slab code is preempt-safe.
62  *  The non-constant members are protected with a per-cache irq spinlock.
63  *
64  * Many thanks to Mark Hemment, who wrote another per-cpu slab patch
65  * in 2000 - many ideas in the current implementation are derived from
66  * his patch.
67  *
68  * Further notes from the original documentation:
69  *
70  * 11 April '97.  Started multi-threading - markhe
71  *	The global cache-chain is protected by the mutex 'slab_mutex'.
72  *	The sem is only needed when accessing/extending the cache-chain, which
73  *	can never happen inside an interrupt (kmem_cache_create(),
74  *	kmem_cache_shrink() and kmem_cache_reap()).
75  *
76  *	At present, each engine can be growing a cache.  This should be blocked.
77  *
78  * 15 March 2005. NUMA slab allocator.
79  *	Shai Fultheim <shai@scalex86.org>.
80  *	Shobhit Dayal <shobhit@calsoftinc.com>
81  *	Alok N Kataria <alokk@calsoftinc.com>
82  *	Christoph Lameter <christoph@lameter.com>
83  *
84  *	Modified the slab allocator to be node aware on NUMA systems.
85  *	Each node has its own list of partial, free and full slabs.
86  *	All object allocations for a node occur from node specific slab lists.
87  */
88 
89 #include	<linux/slab.h>
90 #include	<linux/mm.h>
91 #include	<linux/poison.h>
92 #include	<linux/swap.h>
93 #include	<linux/cache.h>
94 #include	<linux/interrupt.h>
95 #include	<linux/init.h>
96 #include	<linux/compiler.h>
97 #include	<linux/cpuset.h>
98 #include	<linux/proc_fs.h>
99 #include	<linux/seq_file.h>
100 #include	<linux/notifier.h>
101 #include	<linux/kallsyms.h>
102 #include	<linux/cpu.h>
103 #include	<linux/sysctl.h>
104 #include	<linux/module.h>
105 #include	<linux/rcupdate.h>
106 #include	<linux/string.h>
107 #include	<linux/uaccess.h>
108 #include	<linux/nodemask.h>
109 #include	<linux/kmemleak.h>
110 #include	<linux/mempolicy.h>
111 #include	<linux/mutex.h>
112 #include	<linux/fault-inject.h>
113 #include	<linux/rtmutex.h>
114 #include	<linux/reciprocal_div.h>
115 #include	<linux/debugobjects.h>
116 #include	<linux/kmemcheck.h>
117 #include	<linux/memory.h>
118 #include	<linux/prefetch.h>
119 
120 #include	<net/sock.h>
121 
122 #include	<asm/cacheflush.h>
123 #include	<asm/tlbflush.h>
124 #include	<asm/page.h>
125 
126 #include <trace/events/kmem.h>
127 
128 #include	"internal.h"
129 
130 #include	"slab.h"
131 
132 /*
133  * DEBUG	- 1 for kmem_cache_create() to honour; SLAB_RED_ZONE & SLAB_POISON.
134  *		  0 for faster, smaller code (especially in the critical paths).
135  *
136  * STATS	- 1 to collect stats for /proc/slabinfo.
137  *		  0 for faster, smaller code (especially in the critical paths).
138  *
139  * FORCED_DEBUG	- 1 enables SLAB_RED_ZONE and SLAB_POISON (if possible)
140  */
141 
142 #ifdef CONFIG_DEBUG_SLAB
143 #define	DEBUG		1
144 #define	STATS		1
145 #define	FORCED_DEBUG	1
146 #else
147 #define	DEBUG		0
148 #define	STATS		0
149 #define	FORCED_DEBUG	0
150 #endif
151 
152 /* Shouldn't this be in a header file somewhere? */
153 #define	BYTES_PER_WORD		sizeof(void *)
154 #define	REDZONE_ALIGN		max(BYTES_PER_WORD, __alignof__(unsigned long long))
155 
156 #ifndef ARCH_KMALLOC_FLAGS
157 #define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN
158 #endif
159 
160 #define FREELIST_BYTE_INDEX (((PAGE_SIZE >> BITS_PER_BYTE) \
161 				<= SLAB_OBJ_MIN_SIZE) ? 1 : 0)
162 
163 #if FREELIST_BYTE_INDEX
164 typedef unsigned char freelist_idx_t;
165 #else
166 typedef unsigned short freelist_idx_t;
167 #endif
168 
169 #define SLAB_OBJ_MAX_NUM ((1 << sizeof(freelist_idx_t) * BITS_PER_BYTE) - 1)
170 
171 /*
172  * true if a page was allocated from pfmemalloc reserves for network-based
173  * swap
174  */
175 static bool pfmemalloc_active __read_mostly;
176 
177 /*
178  * struct array_cache
179  *
180  * Purpose:
181  * - LIFO ordering, to hand out cache-warm objects from _alloc
182  * - reduce the number of linked list operations
183  * - reduce spinlock operations
184  *
185  * The limit is stored in the per-cpu structure to reduce the data cache
186  * footprint.
187  *
188  */
189 struct array_cache {
190 	unsigned int avail;
191 	unsigned int limit;
192 	unsigned int batchcount;
193 	unsigned int touched;
194 	spinlock_t lock;
195 	void *entry[];	/*
196 			 * Must have this definition in here for the proper
197 			 * alignment of array_cache. Also simplifies accessing
198 			 * the entries.
199 			 *
200 			 * Entries should not be directly dereferenced as
201 			 * entries belonging to slabs marked pfmemalloc will
202 			 * have the lower bits set SLAB_OBJ_PFMEMALLOC
203 			 */
204 };
205 
206 #define SLAB_OBJ_PFMEMALLOC	1
207 static inline bool is_obj_pfmemalloc(void *objp)
208 {
209 	return (unsigned long)objp & SLAB_OBJ_PFMEMALLOC;
210 }
211 
212 static inline void set_obj_pfmemalloc(void **objp)
213 {
214 	*objp = (void *)((unsigned long)*objp | SLAB_OBJ_PFMEMALLOC);
215 	return;
216 }
217 
218 static inline void clear_obj_pfmemalloc(void **objp)
219 {
220 	*objp = (void *)((unsigned long)*objp & ~SLAB_OBJ_PFMEMALLOC);
221 }
222 
223 /*
224  * bootstrap: The caches do not work without cpuarrays anymore, but the
225  * cpuarrays are allocated from the generic caches...
226  */
227 #define BOOT_CPUCACHE_ENTRIES	1
228 struct arraycache_init {
229 	struct array_cache cache;
230 	void *entries[BOOT_CPUCACHE_ENTRIES];
231 };
232 
233 /*
234  * Need this for bootstrapping a per node allocator.
235  */
236 #define NUM_INIT_LISTS (3 * MAX_NUMNODES)
237 static struct kmem_cache_node __initdata init_kmem_cache_node[NUM_INIT_LISTS];
238 #define	CACHE_CACHE 0
239 #define	SIZE_AC MAX_NUMNODES
240 #define	SIZE_NODE (2 * MAX_NUMNODES)
241 
242 static int drain_freelist(struct kmem_cache *cache,
243 			struct kmem_cache_node *n, int tofree);
244 static void free_block(struct kmem_cache *cachep, void **objpp, int len,
245 			int node);
246 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp);
247 static void cache_reap(struct work_struct *unused);
248 
249 static int slab_early_init = 1;
250 
251 #define INDEX_AC kmalloc_index(sizeof(struct arraycache_init))
252 #define INDEX_NODE kmalloc_index(sizeof(struct kmem_cache_node))
253 
254 static void kmem_cache_node_init(struct kmem_cache_node *parent)
255 {
256 	INIT_LIST_HEAD(&parent->slabs_full);
257 	INIT_LIST_HEAD(&parent->slabs_partial);
258 	INIT_LIST_HEAD(&parent->slabs_free);
259 	parent->shared = NULL;
260 	parent->alien = NULL;
261 	parent->colour_next = 0;
262 	spin_lock_init(&parent->list_lock);
263 	parent->free_objects = 0;
264 	parent->free_touched = 0;
265 }
266 
267 #define MAKE_LIST(cachep, listp, slab, nodeid)				\
268 	do {								\
269 		INIT_LIST_HEAD(listp);					\
270 		list_splice(&(cachep->node[nodeid]->slab), listp);	\
271 	} while (0)
272 
273 #define	MAKE_ALL_LISTS(cachep, ptr, nodeid)				\
274 	do {								\
275 	MAKE_LIST((cachep), (&(ptr)->slabs_full), slabs_full, nodeid);	\
276 	MAKE_LIST((cachep), (&(ptr)->slabs_partial), slabs_partial, nodeid); \
277 	MAKE_LIST((cachep), (&(ptr)->slabs_free), slabs_free, nodeid);	\
278 	} while (0)
279 
280 #define CFLGS_OFF_SLAB		(0x80000000UL)
281 #define	OFF_SLAB(x)	((x)->flags & CFLGS_OFF_SLAB)
282 
283 #define BATCHREFILL_LIMIT	16
284 /*
285  * Optimization question: fewer reaps means less probability for unnessary
286  * cpucache drain/refill cycles.
287  *
288  * OTOH the cpuarrays can contain lots of objects,
289  * which could lock up otherwise freeable slabs.
290  */
291 #define REAPTIMEOUT_AC		(2*HZ)
292 #define REAPTIMEOUT_NODE	(4*HZ)
293 
294 #if STATS
295 #define	STATS_INC_ACTIVE(x)	((x)->num_active++)
296 #define	STATS_DEC_ACTIVE(x)	((x)->num_active--)
297 #define	STATS_INC_ALLOCED(x)	((x)->num_allocations++)
298 #define	STATS_INC_GROWN(x)	((x)->grown++)
299 #define	STATS_ADD_REAPED(x,y)	((x)->reaped += (y))
300 #define	STATS_SET_HIGH(x)						\
301 	do {								\
302 		if ((x)->num_active > (x)->high_mark)			\
303 			(x)->high_mark = (x)->num_active;		\
304 	} while (0)
305 #define	STATS_INC_ERR(x)	((x)->errors++)
306 #define	STATS_INC_NODEALLOCS(x)	((x)->node_allocs++)
307 #define	STATS_INC_NODEFREES(x)	((x)->node_frees++)
308 #define STATS_INC_ACOVERFLOW(x)   ((x)->node_overflow++)
309 #define	STATS_SET_FREEABLE(x, i)					\
310 	do {								\
311 		if ((x)->max_freeable < i)				\
312 			(x)->max_freeable = i;				\
313 	} while (0)
314 #define STATS_INC_ALLOCHIT(x)	atomic_inc(&(x)->allochit)
315 #define STATS_INC_ALLOCMISS(x)	atomic_inc(&(x)->allocmiss)
316 #define STATS_INC_FREEHIT(x)	atomic_inc(&(x)->freehit)
317 #define STATS_INC_FREEMISS(x)	atomic_inc(&(x)->freemiss)
318 #else
319 #define	STATS_INC_ACTIVE(x)	do { } while (0)
320 #define	STATS_DEC_ACTIVE(x)	do { } while (0)
321 #define	STATS_INC_ALLOCED(x)	do { } while (0)
322 #define	STATS_INC_GROWN(x)	do { } while (0)
323 #define	STATS_ADD_REAPED(x,y)	do { (void)(y); } while (0)
324 #define	STATS_SET_HIGH(x)	do { } while (0)
325 #define	STATS_INC_ERR(x)	do { } while (0)
326 #define	STATS_INC_NODEALLOCS(x)	do { } while (0)
327 #define	STATS_INC_NODEFREES(x)	do { } while (0)
328 #define STATS_INC_ACOVERFLOW(x)   do { } while (0)
329 #define	STATS_SET_FREEABLE(x, i) do { } while (0)
330 #define STATS_INC_ALLOCHIT(x)	do { } while (0)
331 #define STATS_INC_ALLOCMISS(x)	do { } while (0)
332 #define STATS_INC_FREEHIT(x)	do { } while (0)
333 #define STATS_INC_FREEMISS(x)	do { } while (0)
334 #endif
335 
336 #if DEBUG
337 
338 /*
339  * memory layout of objects:
340  * 0		: objp
341  * 0 .. cachep->obj_offset - BYTES_PER_WORD - 1: padding. This ensures that
342  * 		the end of an object is aligned with the end of the real
343  * 		allocation. Catches writes behind the end of the allocation.
344  * cachep->obj_offset - BYTES_PER_WORD .. cachep->obj_offset - 1:
345  * 		redzone word.
346  * cachep->obj_offset: The real object.
347  * cachep->size - 2* BYTES_PER_WORD: redzone word [BYTES_PER_WORD long]
348  * cachep->size - 1* BYTES_PER_WORD: last caller address
349  *					[BYTES_PER_WORD long]
350  */
351 static int obj_offset(struct kmem_cache *cachep)
352 {
353 	return cachep->obj_offset;
354 }
355 
356 static unsigned long long *dbg_redzone1(struct kmem_cache *cachep, void *objp)
357 {
358 	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
359 	return (unsigned long long*) (objp + obj_offset(cachep) -
360 				      sizeof(unsigned long long));
361 }
362 
363 static unsigned long long *dbg_redzone2(struct kmem_cache *cachep, void *objp)
364 {
365 	BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
366 	if (cachep->flags & SLAB_STORE_USER)
367 		return (unsigned long long *)(objp + cachep->size -
368 					      sizeof(unsigned long long) -
369 					      REDZONE_ALIGN);
370 	return (unsigned long long *) (objp + cachep->size -
371 				       sizeof(unsigned long long));
372 }
373 
374 static void **dbg_userword(struct kmem_cache *cachep, void *objp)
375 {
376 	BUG_ON(!(cachep->flags & SLAB_STORE_USER));
377 	return (void **)(objp + cachep->size - BYTES_PER_WORD);
378 }
379 
380 #else
381 
382 #define obj_offset(x)			0
383 #define dbg_redzone1(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
384 #define dbg_redzone2(cachep, objp)	({BUG(); (unsigned long long *)NULL;})
385 #define dbg_userword(cachep, objp)	({BUG(); (void **)NULL;})
386 
387 #endif
388 
389 /*
390  * Do not go above this order unless 0 objects fit into the slab or
391  * overridden on the command line.
392  */
393 #define	SLAB_MAX_ORDER_HI	1
394 #define	SLAB_MAX_ORDER_LO	0
395 static int slab_max_order = SLAB_MAX_ORDER_LO;
396 static bool slab_max_order_set __initdata;
397 
398 static inline struct kmem_cache *virt_to_cache(const void *obj)
399 {
400 	struct page *page = virt_to_head_page(obj);
401 	return page->slab_cache;
402 }
403 
404 static inline void *index_to_obj(struct kmem_cache *cache, struct page *page,
405 				 unsigned int idx)
406 {
407 	return page->s_mem + cache->size * idx;
408 }
409 
410 /*
411  * We want to avoid an expensive divide : (offset / cache->size)
412  *   Using the fact that size is a constant for a particular cache,
413  *   we can replace (offset / cache->size) by
414  *   reciprocal_divide(offset, cache->reciprocal_buffer_size)
415  */
416 static inline unsigned int obj_to_index(const struct kmem_cache *cache,
417 					const struct page *page, void *obj)
418 {
419 	u32 offset = (obj - page->s_mem);
420 	return reciprocal_divide(offset, cache->reciprocal_buffer_size);
421 }
422 
423 static struct arraycache_init initarray_generic =
424     { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
425 
426 /* internal cache of cache description objs */
427 static struct kmem_cache kmem_cache_boot = {
428 	.batchcount = 1,
429 	.limit = BOOT_CPUCACHE_ENTRIES,
430 	.shared = 1,
431 	.size = sizeof(struct kmem_cache),
432 	.name = "kmem_cache",
433 };
434 
435 #define BAD_ALIEN_MAGIC 0x01020304ul
436 
437 #ifdef CONFIG_LOCKDEP
438 
439 /*
440  * Slab sometimes uses the kmalloc slabs to store the slab headers
441  * for other slabs "off slab".
442  * The locking for this is tricky in that it nests within the locks
443  * of all other slabs in a few places; to deal with this special
444  * locking we put on-slab caches into a separate lock-class.
445  *
446  * We set lock class for alien array caches which are up during init.
447  * The lock annotation will be lost if all cpus of a node goes down and
448  * then comes back up during hotplug
449  */
450 static struct lock_class_key on_slab_l3_key;
451 static struct lock_class_key on_slab_alc_key;
452 
453 static struct lock_class_key debugobj_l3_key;
454 static struct lock_class_key debugobj_alc_key;
455 
456 static void slab_set_lock_classes(struct kmem_cache *cachep,
457 		struct lock_class_key *l3_key, struct lock_class_key *alc_key,
458 		int q)
459 {
460 	struct array_cache **alc;
461 	struct kmem_cache_node *n;
462 	int r;
463 
464 	n = cachep->node[q];
465 	if (!n)
466 		return;
467 
468 	lockdep_set_class(&n->list_lock, l3_key);
469 	alc = n->alien;
470 	/*
471 	 * FIXME: This check for BAD_ALIEN_MAGIC
472 	 * should go away when common slab code is taught to
473 	 * work even without alien caches.
474 	 * Currently, non NUMA code returns BAD_ALIEN_MAGIC
475 	 * for alloc_alien_cache,
476 	 */
477 	if (!alc || (unsigned long)alc == BAD_ALIEN_MAGIC)
478 		return;
479 	for_each_node(r) {
480 		if (alc[r])
481 			lockdep_set_class(&alc[r]->lock, alc_key);
482 	}
483 }
484 
485 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
486 {
487 	slab_set_lock_classes(cachep, &debugobj_l3_key, &debugobj_alc_key, node);
488 }
489 
490 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
491 {
492 	int node;
493 
494 	for_each_online_node(node)
495 		slab_set_debugobj_lock_classes_node(cachep, node);
496 }
497 
498 static void init_node_lock_keys(int q)
499 {
500 	int i;
501 
502 	if (slab_state < UP)
503 		return;
504 
505 	for (i = 1; i <= KMALLOC_SHIFT_HIGH; i++) {
506 		struct kmem_cache_node *n;
507 		struct kmem_cache *cache = kmalloc_caches[i];
508 
509 		if (!cache)
510 			continue;
511 
512 		n = cache->node[q];
513 		if (!n || OFF_SLAB(cache))
514 			continue;
515 
516 		slab_set_lock_classes(cache, &on_slab_l3_key,
517 				&on_slab_alc_key, q);
518 	}
519 }
520 
521 static void on_slab_lock_classes_node(struct kmem_cache *cachep, int q)
522 {
523 	if (!cachep->node[q])
524 		return;
525 
526 	slab_set_lock_classes(cachep, &on_slab_l3_key,
527 			&on_slab_alc_key, q);
528 }
529 
530 static inline void on_slab_lock_classes(struct kmem_cache *cachep)
531 {
532 	int node;
533 
534 	VM_BUG_ON(OFF_SLAB(cachep));
535 	for_each_node(node)
536 		on_slab_lock_classes_node(cachep, node);
537 }
538 
539 static inline void init_lock_keys(void)
540 {
541 	int node;
542 
543 	for_each_node(node)
544 		init_node_lock_keys(node);
545 }
546 #else
547 static void init_node_lock_keys(int q)
548 {
549 }
550 
551 static inline void init_lock_keys(void)
552 {
553 }
554 
555 static inline void on_slab_lock_classes(struct kmem_cache *cachep)
556 {
557 }
558 
559 static inline void on_slab_lock_classes_node(struct kmem_cache *cachep, int node)
560 {
561 }
562 
563 static void slab_set_debugobj_lock_classes_node(struct kmem_cache *cachep, int node)
564 {
565 }
566 
567 static void slab_set_debugobj_lock_classes(struct kmem_cache *cachep)
568 {
569 }
570 #endif
571 
572 static DEFINE_PER_CPU(struct delayed_work, slab_reap_work);
573 
574 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
575 {
576 	return cachep->array[smp_processor_id()];
577 }
578 
579 static int calculate_nr_objs(size_t slab_size, size_t buffer_size,
580 				size_t idx_size, size_t align)
581 {
582 	int nr_objs;
583 	size_t freelist_size;
584 
585 	/*
586 	 * Ignore padding for the initial guess. The padding
587 	 * is at most @align-1 bytes, and @buffer_size is at
588 	 * least @align. In the worst case, this result will
589 	 * be one greater than the number of objects that fit
590 	 * into the memory allocation when taking the padding
591 	 * into account.
592 	 */
593 	nr_objs = slab_size / (buffer_size + idx_size);
594 
595 	/*
596 	 * This calculated number will be either the right
597 	 * amount, or one greater than what we want.
598 	 */
599 	freelist_size = slab_size - nr_objs * buffer_size;
600 	if (freelist_size < ALIGN(nr_objs * idx_size, align))
601 		nr_objs--;
602 
603 	return nr_objs;
604 }
605 
606 /*
607  * Calculate the number of objects and left-over bytes for a given buffer size.
608  */
609 static void cache_estimate(unsigned long gfporder, size_t buffer_size,
610 			   size_t align, int flags, size_t *left_over,
611 			   unsigned int *num)
612 {
613 	int nr_objs;
614 	size_t mgmt_size;
615 	size_t slab_size = PAGE_SIZE << gfporder;
616 
617 	/*
618 	 * The slab management structure can be either off the slab or
619 	 * on it. For the latter case, the memory allocated for a
620 	 * slab is used for:
621 	 *
622 	 * - One unsigned int for each object
623 	 * - Padding to respect alignment of @align
624 	 * - @buffer_size bytes for each object
625 	 *
626 	 * If the slab management structure is off the slab, then the
627 	 * alignment will already be calculated into the size. Because
628 	 * the slabs are all pages aligned, the objects will be at the
629 	 * correct alignment when allocated.
630 	 */
631 	if (flags & CFLGS_OFF_SLAB) {
632 		mgmt_size = 0;
633 		nr_objs = slab_size / buffer_size;
634 
635 	} else {
636 		nr_objs = calculate_nr_objs(slab_size, buffer_size,
637 					sizeof(freelist_idx_t), align);
638 		mgmt_size = ALIGN(nr_objs * sizeof(freelist_idx_t), align);
639 	}
640 	*num = nr_objs;
641 	*left_over = slab_size - nr_objs*buffer_size - mgmt_size;
642 }
643 
644 #if DEBUG
645 #define slab_error(cachep, msg) __slab_error(__func__, cachep, msg)
646 
647 static void __slab_error(const char *function, struct kmem_cache *cachep,
648 			char *msg)
649 {
650 	printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
651 	       function, cachep->name, msg);
652 	dump_stack();
653 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
654 }
655 #endif
656 
657 /*
658  * By default on NUMA we use alien caches to stage the freeing of
659  * objects allocated from other nodes. This causes massive memory
660  * inefficiencies when using fake NUMA setup to split memory into a
661  * large number of small nodes, so it can be disabled on the command
662  * line
663   */
664 
665 static int use_alien_caches __read_mostly = 1;
666 static int __init noaliencache_setup(char *s)
667 {
668 	use_alien_caches = 0;
669 	return 1;
670 }
671 __setup("noaliencache", noaliencache_setup);
672 
673 static int __init slab_max_order_setup(char *str)
674 {
675 	get_option(&str, &slab_max_order);
676 	slab_max_order = slab_max_order < 0 ? 0 :
677 				min(slab_max_order, MAX_ORDER - 1);
678 	slab_max_order_set = true;
679 
680 	return 1;
681 }
682 __setup("slab_max_order=", slab_max_order_setup);
683 
684 #ifdef CONFIG_NUMA
685 /*
686  * Special reaping functions for NUMA systems called from cache_reap().
687  * These take care of doing round robin flushing of alien caches (containing
688  * objects freed on different nodes from which they were allocated) and the
689  * flushing of remote pcps by calling drain_node_pages.
690  */
691 static DEFINE_PER_CPU(unsigned long, slab_reap_node);
692 
693 static void init_reap_node(int cpu)
694 {
695 	int node;
696 
697 	node = next_node(cpu_to_mem(cpu), node_online_map);
698 	if (node == MAX_NUMNODES)
699 		node = first_node(node_online_map);
700 
701 	per_cpu(slab_reap_node, cpu) = node;
702 }
703 
704 static void next_reap_node(void)
705 {
706 	int node = __this_cpu_read(slab_reap_node);
707 
708 	node = next_node(node, node_online_map);
709 	if (unlikely(node >= MAX_NUMNODES))
710 		node = first_node(node_online_map);
711 	__this_cpu_write(slab_reap_node, node);
712 }
713 
714 #else
715 #define init_reap_node(cpu) do { } while (0)
716 #define next_reap_node(void) do { } while (0)
717 #endif
718 
719 /*
720  * Initiate the reap timer running on the target CPU.  We run at around 1 to 2Hz
721  * via the workqueue/eventd.
722  * Add the CPU number into the expiration time to minimize the possibility of
723  * the CPUs getting into lockstep and contending for the global cache chain
724  * lock.
725  */
726 static void start_cpu_timer(int cpu)
727 {
728 	struct delayed_work *reap_work = &per_cpu(slab_reap_work, cpu);
729 
730 	/*
731 	 * When this gets called from do_initcalls via cpucache_init(),
732 	 * init_workqueues() has already run, so keventd will be setup
733 	 * at that time.
734 	 */
735 	if (keventd_up() && reap_work->work.func == NULL) {
736 		init_reap_node(cpu);
737 		INIT_DEFERRABLE_WORK(reap_work, cache_reap);
738 		schedule_delayed_work_on(cpu, reap_work,
739 					__round_jiffies_relative(HZ, cpu));
740 	}
741 }
742 
743 static struct array_cache *alloc_arraycache(int node, int entries,
744 					    int batchcount, gfp_t gfp)
745 {
746 	int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
747 	struct array_cache *nc = NULL;
748 
749 	nc = kmalloc_node(memsize, gfp, node);
750 	/*
751 	 * The array_cache structures contain pointers to free object.
752 	 * However, when such objects are allocated or transferred to another
753 	 * cache the pointers are not cleared and they could be counted as
754 	 * valid references during a kmemleak scan. Therefore, kmemleak must
755 	 * not scan such objects.
756 	 */
757 	kmemleak_no_scan(nc);
758 	if (nc) {
759 		nc->avail = 0;
760 		nc->limit = entries;
761 		nc->batchcount = batchcount;
762 		nc->touched = 0;
763 		spin_lock_init(&nc->lock);
764 	}
765 	return nc;
766 }
767 
768 static inline bool is_slab_pfmemalloc(struct page *page)
769 {
770 	return PageSlabPfmemalloc(page);
771 }
772 
773 /* Clears pfmemalloc_active if no slabs have pfmalloc set */
774 static void recheck_pfmemalloc_active(struct kmem_cache *cachep,
775 						struct array_cache *ac)
776 {
777 	struct kmem_cache_node *n = cachep->node[numa_mem_id()];
778 	struct page *page;
779 	unsigned long flags;
780 
781 	if (!pfmemalloc_active)
782 		return;
783 
784 	spin_lock_irqsave(&n->list_lock, flags);
785 	list_for_each_entry(page, &n->slabs_full, lru)
786 		if (is_slab_pfmemalloc(page))
787 			goto out;
788 
789 	list_for_each_entry(page, &n->slabs_partial, lru)
790 		if (is_slab_pfmemalloc(page))
791 			goto out;
792 
793 	list_for_each_entry(page, &n->slabs_free, lru)
794 		if (is_slab_pfmemalloc(page))
795 			goto out;
796 
797 	pfmemalloc_active = false;
798 out:
799 	spin_unlock_irqrestore(&n->list_lock, flags);
800 }
801 
802 static void *__ac_get_obj(struct kmem_cache *cachep, struct array_cache *ac,
803 						gfp_t flags, bool force_refill)
804 {
805 	int i;
806 	void *objp = ac->entry[--ac->avail];
807 
808 	/* Ensure the caller is allowed to use objects from PFMEMALLOC slab */
809 	if (unlikely(is_obj_pfmemalloc(objp))) {
810 		struct kmem_cache_node *n;
811 
812 		if (gfp_pfmemalloc_allowed(flags)) {
813 			clear_obj_pfmemalloc(&objp);
814 			return objp;
815 		}
816 
817 		/* The caller cannot use PFMEMALLOC objects, find another one */
818 		for (i = 0; i < ac->avail; i++) {
819 			/* If a !PFMEMALLOC object is found, swap them */
820 			if (!is_obj_pfmemalloc(ac->entry[i])) {
821 				objp = ac->entry[i];
822 				ac->entry[i] = ac->entry[ac->avail];
823 				ac->entry[ac->avail] = objp;
824 				return objp;
825 			}
826 		}
827 
828 		/*
829 		 * If there are empty slabs on the slabs_free list and we are
830 		 * being forced to refill the cache, mark this one !pfmemalloc.
831 		 */
832 		n = cachep->node[numa_mem_id()];
833 		if (!list_empty(&n->slabs_free) && force_refill) {
834 			struct page *page = virt_to_head_page(objp);
835 			ClearPageSlabPfmemalloc(page);
836 			clear_obj_pfmemalloc(&objp);
837 			recheck_pfmemalloc_active(cachep, ac);
838 			return objp;
839 		}
840 
841 		/* No !PFMEMALLOC objects available */
842 		ac->avail++;
843 		objp = NULL;
844 	}
845 
846 	return objp;
847 }
848 
849 static inline void *ac_get_obj(struct kmem_cache *cachep,
850 			struct array_cache *ac, gfp_t flags, bool force_refill)
851 {
852 	void *objp;
853 
854 	if (unlikely(sk_memalloc_socks()))
855 		objp = __ac_get_obj(cachep, ac, flags, force_refill);
856 	else
857 		objp = ac->entry[--ac->avail];
858 
859 	return objp;
860 }
861 
862 static void *__ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
863 								void *objp)
864 {
865 	if (unlikely(pfmemalloc_active)) {
866 		/* Some pfmemalloc slabs exist, check if this is one */
867 		struct page *page = virt_to_head_page(objp);
868 		if (PageSlabPfmemalloc(page))
869 			set_obj_pfmemalloc(&objp);
870 	}
871 
872 	return objp;
873 }
874 
875 static inline void ac_put_obj(struct kmem_cache *cachep, struct array_cache *ac,
876 								void *objp)
877 {
878 	if (unlikely(sk_memalloc_socks()))
879 		objp = __ac_put_obj(cachep, ac, objp);
880 
881 	ac->entry[ac->avail++] = objp;
882 }
883 
884 /*
885  * Transfer objects in one arraycache to another.
886  * Locking must be handled by the caller.
887  *
888  * Return the number of entries transferred.
889  */
890 static int transfer_objects(struct array_cache *to,
891 		struct array_cache *from, unsigned int max)
892 {
893 	/* Figure out how many entries to transfer */
894 	int nr = min3(from->avail, max, to->limit - to->avail);
895 
896 	if (!nr)
897 		return 0;
898 
899 	memcpy(to->entry + to->avail, from->entry + from->avail -nr,
900 			sizeof(void *) *nr);
901 
902 	from->avail -= nr;
903 	to->avail += nr;
904 	return nr;
905 }
906 
907 #ifndef CONFIG_NUMA
908 
909 #define drain_alien_cache(cachep, alien) do { } while (0)
910 #define reap_alien(cachep, n) do { } while (0)
911 
912 static inline struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
913 {
914 	return (struct array_cache **)BAD_ALIEN_MAGIC;
915 }
916 
917 static inline void free_alien_cache(struct array_cache **ac_ptr)
918 {
919 }
920 
921 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
922 {
923 	return 0;
924 }
925 
926 static inline void *alternate_node_alloc(struct kmem_cache *cachep,
927 		gfp_t flags)
928 {
929 	return NULL;
930 }
931 
932 static inline void *____cache_alloc_node(struct kmem_cache *cachep,
933 		 gfp_t flags, int nodeid)
934 {
935 	return NULL;
936 }
937 
938 #else	/* CONFIG_NUMA */
939 
940 static void *____cache_alloc_node(struct kmem_cache *, gfp_t, int);
941 static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
942 
943 static struct array_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
944 {
945 	struct array_cache **ac_ptr;
946 	int memsize = sizeof(void *) * nr_node_ids;
947 	int i;
948 
949 	if (limit > 1)
950 		limit = 12;
951 	ac_ptr = kzalloc_node(memsize, gfp, node);
952 	if (ac_ptr) {
953 		for_each_node(i) {
954 			if (i == node || !node_online(i))
955 				continue;
956 			ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d, gfp);
957 			if (!ac_ptr[i]) {
958 				for (i--; i >= 0; i--)
959 					kfree(ac_ptr[i]);
960 				kfree(ac_ptr);
961 				return NULL;
962 			}
963 		}
964 	}
965 	return ac_ptr;
966 }
967 
968 static void free_alien_cache(struct array_cache **ac_ptr)
969 {
970 	int i;
971 
972 	if (!ac_ptr)
973 		return;
974 	for_each_node(i)
975 	    kfree(ac_ptr[i]);
976 	kfree(ac_ptr);
977 }
978 
979 static void __drain_alien_cache(struct kmem_cache *cachep,
980 				struct array_cache *ac, int node)
981 {
982 	struct kmem_cache_node *n = cachep->node[node];
983 
984 	if (ac->avail) {
985 		spin_lock(&n->list_lock);
986 		/*
987 		 * Stuff objects into the remote nodes shared array first.
988 		 * That way we could avoid the overhead of putting the objects
989 		 * into the free lists and getting them back later.
990 		 */
991 		if (n->shared)
992 			transfer_objects(n->shared, ac, ac->limit);
993 
994 		free_block(cachep, ac->entry, ac->avail, node);
995 		ac->avail = 0;
996 		spin_unlock(&n->list_lock);
997 	}
998 }
999 
1000 /*
1001  * Called from cache_reap() to regularly drain alien caches round robin.
1002  */
1003 static void reap_alien(struct kmem_cache *cachep, struct kmem_cache_node *n)
1004 {
1005 	int node = __this_cpu_read(slab_reap_node);
1006 
1007 	if (n->alien) {
1008 		struct array_cache *ac = n->alien[node];
1009 
1010 		if (ac && ac->avail && spin_trylock_irq(&ac->lock)) {
1011 			__drain_alien_cache(cachep, ac, node);
1012 			spin_unlock_irq(&ac->lock);
1013 		}
1014 	}
1015 }
1016 
1017 static void drain_alien_cache(struct kmem_cache *cachep,
1018 				struct array_cache **alien)
1019 {
1020 	int i = 0;
1021 	struct array_cache *ac;
1022 	unsigned long flags;
1023 
1024 	for_each_online_node(i) {
1025 		ac = alien[i];
1026 		if (ac) {
1027 			spin_lock_irqsave(&ac->lock, flags);
1028 			__drain_alien_cache(cachep, ac, i);
1029 			spin_unlock_irqrestore(&ac->lock, flags);
1030 		}
1031 	}
1032 }
1033 
1034 static inline int cache_free_alien(struct kmem_cache *cachep, void *objp)
1035 {
1036 	int nodeid = page_to_nid(virt_to_page(objp));
1037 	struct kmem_cache_node *n;
1038 	struct array_cache *alien = NULL;
1039 	int node;
1040 
1041 	node = numa_mem_id();
1042 
1043 	/*
1044 	 * Make sure we are not freeing a object from another node to the array
1045 	 * cache on this cpu.
1046 	 */
1047 	if (likely(nodeid == node))
1048 		return 0;
1049 
1050 	n = cachep->node[node];
1051 	STATS_INC_NODEFREES(cachep);
1052 	if (n->alien && n->alien[nodeid]) {
1053 		alien = n->alien[nodeid];
1054 		spin_lock(&alien->lock);
1055 		if (unlikely(alien->avail == alien->limit)) {
1056 			STATS_INC_ACOVERFLOW(cachep);
1057 			__drain_alien_cache(cachep, alien, nodeid);
1058 		}
1059 		ac_put_obj(cachep, alien, objp);
1060 		spin_unlock(&alien->lock);
1061 	} else {
1062 		spin_lock(&(cachep->node[nodeid])->list_lock);
1063 		free_block(cachep, &objp, 1, nodeid);
1064 		spin_unlock(&(cachep->node[nodeid])->list_lock);
1065 	}
1066 	return 1;
1067 }
1068 #endif
1069 
1070 /*
1071  * Allocates and initializes node for a node on each slab cache, used for
1072  * either memory or cpu hotplug.  If memory is being hot-added, the kmem_cache_node
1073  * will be allocated off-node since memory is not yet online for the new node.
1074  * When hotplugging memory or a cpu, existing node are not replaced if
1075  * already in use.
1076  *
1077  * Must hold slab_mutex.
1078  */
1079 static int init_cache_node_node(int node)
1080 {
1081 	struct kmem_cache *cachep;
1082 	struct kmem_cache_node *n;
1083 	const int memsize = sizeof(struct kmem_cache_node);
1084 
1085 	list_for_each_entry(cachep, &slab_caches, list) {
1086 		/*
1087 		 * Set up the kmem_cache_node for cpu before we can
1088 		 * begin anything. Make sure some other cpu on this
1089 		 * node has not already allocated this
1090 		 */
1091 		if (!cachep->node[node]) {
1092 			n = kmalloc_node(memsize, GFP_KERNEL, node);
1093 			if (!n)
1094 				return -ENOMEM;
1095 			kmem_cache_node_init(n);
1096 			n->next_reap = jiffies + REAPTIMEOUT_NODE +
1097 			    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1098 
1099 			/*
1100 			 * The kmem_cache_nodes don't come and go as CPUs
1101 			 * come and go.  slab_mutex is sufficient
1102 			 * protection here.
1103 			 */
1104 			cachep->node[node] = n;
1105 		}
1106 
1107 		spin_lock_irq(&cachep->node[node]->list_lock);
1108 		cachep->node[node]->free_limit =
1109 			(1 + nr_cpus_node(node)) *
1110 			cachep->batchcount + cachep->num;
1111 		spin_unlock_irq(&cachep->node[node]->list_lock);
1112 	}
1113 	return 0;
1114 }
1115 
1116 static inline int slabs_tofree(struct kmem_cache *cachep,
1117 						struct kmem_cache_node *n)
1118 {
1119 	return (n->free_objects + cachep->num - 1) / cachep->num;
1120 }
1121 
1122 static void cpuup_canceled(long cpu)
1123 {
1124 	struct kmem_cache *cachep;
1125 	struct kmem_cache_node *n = NULL;
1126 	int node = cpu_to_mem(cpu);
1127 	const struct cpumask *mask = cpumask_of_node(node);
1128 
1129 	list_for_each_entry(cachep, &slab_caches, list) {
1130 		struct array_cache *nc;
1131 		struct array_cache *shared;
1132 		struct array_cache **alien;
1133 
1134 		/* cpu is dead; no one can alloc from it. */
1135 		nc = cachep->array[cpu];
1136 		cachep->array[cpu] = NULL;
1137 		n = cachep->node[node];
1138 
1139 		if (!n)
1140 			goto free_array_cache;
1141 
1142 		spin_lock_irq(&n->list_lock);
1143 
1144 		/* Free limit for this kmem_cache_node */
1145 		n->free_limit -= cachep->batchcount;
1146 		if (nc)
1147 			free_block(cachep, nc->entry, nc->avail, node);
1148 
1149 		if (!cpumask_empty(mask)) {
1150 			spin_unlock_irq(&n->list_lock);
1151 			goto free_array_cache;
1152 		}
1153 
1154 		shared = n->shared;
1155 		if (shared) {
1156 			free_block(cachep, shared->entry,
1157 				   shared->avail, node);
1158 			n->shared = NULL;
1159 		}
1160 
1161 		alien = n->alien;
1162 		n->alien = NULL;
1163 
1164 		spin_unlock_irq(&n->list_lock);
1165 
1166 		kfree(shared);
1167 		if (alien) {
1168 			drain_alien_cache(cachep, alien);
1169 			free_alien_cache(alien);
1170 		}
1171 free_array_cache:
1172 		kfree(nc);
1173 	}
1174 	/*
1175 	 * In the previous loop, all the objects were freed to
1176 	 * the respective cache's slabs,  now we can go ahead and
1177 	 * shrink each nodelist to its limit.
1178 	 */
1179 	list_for_each_entry(cachep, &slab_caches, list) {
1180 		n = cachep->node[node];
1181 		if (!n)
1182 			continue;
1183 		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1184 	}
1185 }
1186 
1187 static int cpuup_prepare(long cpu)
1188 {
1189 	struct kmem_cache *cachep;
1190 	struct kmem_cache_node *n = NULL;
1191 	int node = cpu_to_mem(cpu);
1192 	int err;
1193 
1194 	/*
1195 	 * We need to do this right in the beginning since
1196 	 * alloc_arraycache's are going to use this list.
1197 	 * kmalloc_node allows us to add the slab to the right
1198 	 * kmem_cache_node and not this cpu's kmem_cache_node
1199 	 */
1200 	err = init_cache_node_node(node);
1201 	if (err < 0)
1202 		goto bad;
1203 
1204 	/*
1205 	 * Now we can go ahead with allocating the shared arrays and
1206 	 * array caches
1207 	 */
1208 	list_for_each_entry(cachep, &slab_caches, list) {
1209 		struct array_cache *nc;
1210 		struct array_cache *shared = NULL;
1211 		struct array_cache **alien = NULL;
1212 
1213 		nc = alloc_arraycache(node, cachep->limit,
1214 					cachep->batchcount, GFP_KERNEL);
1215 		if (!nc)
1216 			goto bad;
1217 		if (cachep->shared) {
1218 			shared = alloc_arraycache(node,
1219 				cachep->shared * cachep->batchcount,
1220 				0xbaadf00d, GFP_KERNEL);
1221 			if (!shared) {
1222 				kfree(nc);
1223 				goto bad;
1224 			}
1225 		}
1226 		if (use_alien_caches) {
1227 			alien = alloc_alien_cache(node, cachep->limit, GFP_KERNEL);
1228 			if (!alien) {
1229 				kfree(shared);
1230 				kfree(nc);
1231 				goto bad;
1232 			}
1233 		}
1234 		cachep->array[cpu] = nc;
1235 		n = cachep->node[node];
1236 		BUG_ON(!n);
1237 
1238 		spin_lock_irq(&n->list_lock);
1239 		if (!n->shared) {
1240 			/*
1241 			 * We are serialised from CPU_DEAD or
1242 			 * CPU_UP_CANCELLED by the cpucontrol lock
1243 			 */
1244 			n->shared = shared;
1245 			shared = NULL;
1246 		}
1247 #ifdef CONFIG_NUMA
1248 		if (!n->alien) {
1249 			n->alien = alien;
1250 			alien = NULL;
1251 		}
1252 #endif
1253 		spin_unlock_irq(&n->list_lock);
1254 		kfree(shared);
1255 		free_alien_cache(alien);
1256 		if (cachep->flags & SLAB_DEBUG_OBJECTS)
1257 			slab_set_debugobj_lock_classes_node(cachep, node);
1258 		else if (!OFF_SLAB(cachep) &&
1259 			 !(cachep->flags & SLAB_DESTROY_BY_RCU))
1260 			on_slab_lock_classes_node(cachep, node);
1261 	}
1262 	init_node_lock_keys(node);
1263 
1264 	return 0;
1265 bad:
1266 	cpuup_canceled(cpu);
1267 	return -ENOMEM;
1268 }
1269 
1270 static int cpuup_callback(struct notifier_block *nfb,
1271 				    unsigned long action, void *hcpu)
1272 {
1273 	long cpu = (long)hcpu;
1274 	int err = 0;
1275 
1276 	switch (action) {
1277 	case CPU_UP_PREPARE:
1278 	case CPU_UP_PREPARE_FROZEN:
1279 		mutex_lock(&slab_mutex);
1280 		err = cpuup_prepare(cpu);
1281 		mutex_unlock(&slab_mutex);
1282 		break;
1283 	case CPU_ONLINE:
1284 	case CPU_ONLINE_FROZEN:
1285 		start_cpu_timer(cpu);
1286 		break;
1287 #ifdef CONFIG_HOTPLUG_CPU
1288   	case CPU_DOWN_PREPARE:
1289   	case CPU_DOWN_PREPARE_FROZEN:
1290 		/*
1291 		 * Shutdown cache reaper. Note that the slab_mutex is
1292 		 * held so that if cache_reap() is invoked it cannot do
1293 		 * anything expensive but will only modify reap_work
1294 		 * and reschedule the timer.
1295 		*/
1296 		cancel_delayed_work_sync(&per_cpu(slab_reap_work, cpu));
1297 		/* Now the cache_reaper is guaranteed to be not running. */
1298 		per_cpu(slab_reap_work, cpu).work.func = NULL;
1299   		break;
1300   	case CPU_DOWN_FAILED:
1301   	case CPU_DOWN_FAILED_FROZEN:
1302 		start_cpu_timer(cpu);
1303   		break;
1304 	case CPU_DEAD:
1305 	case CPU_DEAD_FROZEN:
1306 		/*
1307 		 * Even if all the cpus of a node are down, we don't free the
1308 		 * kmem_cache_node of any cache. This to avoid a race between
1309 		 * cpu_down, and a kmalloc allocation from another cpu for
1310 		 * memory from the node of the cpu going down.  The node
1311 		 * structure is usually allocated from kmem_cache_create() and
1312 		 * gets destroyed at kmem_cache_destroy().
1313 		 */
1314 		/* fall through */
1315 #endif
1316 	case CPU_UP_CANCELED:
1317 	case CPU_UP_CANCELED_FROZEN:
1318 		mutex_lock(&slab_mutex);
1319 		cpuup_canceled(cpu);
1320 		mutex_unlock(&slab_mutex);
1321 		break;
1322 	}
1323 	return notifier_from_errno(err);
1324 }
1325 
1326 static struct notifier_block cpucache_notifier = {
1327 	&cpuup_callback, NULL, 0
1328 };
1329 
1330 #if defined(CONFIG_NUMA) && defined(CONFIG_MEMORY_HOTPLUG)
1331 /*
1332  * Drains freelist for a node on each slab cache, used for memory hot-remove.
1333  * Returns -EBUSY if all objects cannot be drained so that the node is not
1334  * removed.
1335  *
1336  * Must hold slab_mutex.
1337  */
1338 static int __meminit drain_cache_node_node(int node)
1339 {
1340 	struct kmem_cache *cachep;
1341 	int ret = 0;
1342 
1343 	list_for_each_entry(cachep, &slab_caches, list) {
1344 		struct kmem_cache_node *n;
1345 
1346 		n = cachep->node[node];
1347 		if (!n)
1348 			continue;
1349 
1350 		drain_freelist(cachep, n, slabs_tofree(cachep, n));
1351 
1352 		if (!list_empty(&n->slabs_full) ||
1353 		    !list_empty(&n->slabs_partial)) {
1354 			ret = -EBUSY;
1355 			break;
1356 		}
1357 	}
1358 	return ret;
1359 }
1360 
1361 static int __meminit slab_memory_callback(struct notifier_block *self,
1362 					unsigned long action, void *arg)
1363 {
1364 	struct memory_notify *mnb = arg;
1365 	int ret = 0;
1366 	int nid;
1367 
1368 	nid = mnb->status_change_nid;
1369 	if (nid < 0)
1370 		goto out;
1371 
1372 	switch (action) {
1373 	case MEM_GOING_ONLINE:
1374 		mutex_lock(&slab_mutex);
1375 		ret = init_cache_node_node(nid);
1376 		mutex_unlock(&slab_mutex);
1377 		break;
1378 	case MEM_GOING_OFFLINE:
1379 		mutex_lock(&slab_mutex);
1380 		ret = drain_cache_node_node(nid);
1381 		mutex_unlock(&slab_mutex);
1382 		break;
1383 	case MEM_ONLINE:
1384 	case MEM_OFFLINE:
1385 	case MEM_CANCEL_ONLINE:
1386 	case MEM_CANCEL_OFFLINE:
1387 		break;
1388 	}
1389 out:
1390 	return notifier_from_errno(ret);
1391 }
1392 #endif /* CONFIG_NUMA && CONFIG_MEMORY_HOTPLUG */
1393 
1394 /*
1395  * swap the static kmem_cache_node with kmalloced memory
1396  */
1397 static void __init init_list(struct kmem_cache *cachep, struct kmem_cache_node *list,
1398 				int nodeid)
1399 {
1400 	struct kmem_cache_node *ptr;
1401 
1402 	ptr = kmalloc_node(sizeof(struct kmem_cache_node), GFP_NOWAIT, nodeid);
1403 	BUG_ON(!ptr);
1404 
1405 	memcpy(ptr, list, sizeof(struct kmem_cache_node));
1406 	/*
1407 	 * Do not assume that spinlocks can be initialized via memcpy:
1408 	 */
1409 	spin_lock_init(&ptr->list_lock);
1410 
1411 	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1412 	cachep->node[nodeid] = ptr;
1413 }
1414 
1415 /*
1416  * For setting up all the kmem_cache_node for cache whose buffer_size is same as
1417  * size of kmem_cache_node.
1418  */
1419 static void __init set_up_node(struct kmem_cache *cachep, int index)
1420 {
1421 	int node;
1422 
1423 	for_each_online_node(node) {
1424 		cachep->node[node] = &init_kmem_cache_node[index + node];
1425 		cachep->node[node]->next_reap = jiffies +
1426 		    REAPTIMEOUT_NODE +
1427 		    ((unsigned long)cachep) % REAPTIMEOUT_NODE;
1428 	}
1429 }
1430 
1431 /*
1432  * The memory after the last cpu cache pointer is used for the
1433  * the node pointer.
1434  */
1435 static void setup_node_pointer(struct kmem_cache *cachep)
1436 {
1437 	cachep->node = (struct kmem_cache_node **)&cachep->array[nr_cpu_ids];
1438 }
1439 
1440 /*
1441  * Initialisation.  Called after the page allocator have been initialised and
1442  * before smp_init().
1443  */
1444 void __init kmem_cache_init(void)
1445 {
1446 	int i;
1447 
1448 	BUILD_BUG_ON(sizeof(((struct page *)NULL)->lru) <
1449 					sizeof(struct rcu_head));
1450 	kmem_cache = &kmem_cache_boot;
1451 	setup_node_pointer(kmem_cache);
1452 
1453 	if (num_possible_nodes() == 1)
1454 		use_alien_caches = 0;
1455 
1456 	for (i = 0; i < NUM_INIT_LISTS; i++)
1457 		kmem_cache_node_init(&init_kmem_cache_node[i]);
1458 
1459 	set_up_node(kmem_cache, CACHE_CACHE);
1460 
1461 	/*
1462 	 * Fragmentation resistance on low memory - only use bigger
1463 	 * page orders on machines with more than 32MB of memory if
1464 	 * not overridden on the command line.
1465 	 */
1466 	if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
1467 		slab_max_order = SLAB_MAX_ORDER_HI;
1468 
1469 	/* Bootstrap is tricky, because several objects are allocated
1470 	 * from caches that do not exist yet:
1471 	 * 1) initialize the kmem_cache cache: it contains the struct
1472 	 *    kmem_cache structures of all caches, except kmem_cache itself:
1473 	 *    kmem_cache is statically allocated.
1474 	 *    Initially an __init data area is used for the head array and the
1475 	 *    kmem_cache_node structures, it's replaced with a kmalloc allocated
1476 	 *    array at the end of the bootstrap.
1477 	 * 2) Create the first kmalloc cache.
1478 	 *    The struct kmem_cache for the new cache is allocated normally.
1479 	 *    An __init data area is used for the head array.
1480 	 * 3) Create the remaining kmalloc caches, with minimally sized
1481 	 *    head arrays.
1482 	 * 4) Replace the __init data head arrays for kmem_cache and the first
1483 	 *    kmalloc cache with kmalloc allocated arrays.
1484 	 * 5) Replace the __init data for kmem_cache_node for kmem_cache and
1485 	 *    the other cache's with kmalloc allocated memory.
1486 	 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
1487 	 */
1488 
1489 	/* 1) create the kmem_cache */
1490 
1491 	/*
1492 	 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1493 	 */
1494 	create_boot_cache(kmem_cache, "kmem_cache",
1495 		offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1496 				  nr_node_ids * sizeof(struct kmem_cache_node *),
1497 				  SLAB_HWCACHE_ALIGN);
1498 	list_add(&kmem_cache->list, &slab_caches);
1499 
1500 	/* 2+3) create the kmalloc caches */
1501 
1502 	/*
1503 	 * Initialize the caches that provide memory for the array cache and the
1504 	 * kmem_cache_node structures first.  Without this, further allocations will
1505 	 * bug.
1506 	 */
1507 
1508 	kmalloc_caches[INDEX_AC] = create_kmalloc_cache("kmalloc-ac",
1509 					kmalloc_size(INDEX_AC), ARCH_KMALLOC_FLAGS);
1510 
1511 	if (INDEX_AC != INDEX_NODE)
1512 		kmalloc_caches[INDEX_NODE] =
1513 			create_kmalloc_cache("kmalloc-node",
1514 				kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS);
1515 
1516 	slab_early_init = 0;
1517 
1518 	/* 4) Replace the bootstrap head arrays */
1519 	{
1520 		struct array_cache *ptr;
1521 
1522 		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1523 
1524 		memcpy(ptr, cpu_cache_get(kmem_cache),
1525 		       sizeof(struct arraycache_init));
1526 		/*
1527 		 * Do not assume that spinlocks can be initialized via memcpy:
1528 		 */
1529 		spin_lock_init(&ptr->lock);
1530 
1531 		kmem_cache->array[smp_processor_id()] = ptr;
1532 
1533 		ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1534 
1535 		BUG_ON(cpu_cache_get(kmalloc_caches[INDEX_AC])
1536 		       != &initarray_generic.cache);
1537 		memcpy(ptr, cpu_cache_get(kmalloc_caches[INDEX_AC]),
1538 		       sizeof(struct arraycache_init));
1539 		/*
1540 		 * Do not assume that spinlocks can be initialized via memcpy:
1541 		 */
1542 		spin_lock_init(&ptr->lock);
1543 
1544 		kmalloc_caches[INDEX_AC]->array[smp_processor_id()] = ptr;
1545 	}
1546 	/* 5) Replace the bootstrap kmem_cache_node */
1547 	{
1548 		int nid;
1549 
1550 		for_each_online_node(nid) {
1551 			init_list(kmem_cache, &init_kmem_cache_node[CACHE_CACHE + nid], nid);
1552 
1553 			init_list(kmalloc_caches[INDEX_AC],
1554 				  &init_kmem_cache_node[SIZE_AC + nid], nid);
1555 
1556 			if (INDEX_AC != INDEX_NODE) {
1557 				init_list(kmalloc_caches[INDEX_NODE],
1558 					  &init_kmem_cache_node[SIZE_NODE + nid], nid);
1559 			}
1560 		}
1561 	}
1562 
1563 	create_kmalloc_caches(ARCH_KMALLOC_FLAGS);
1564 }
1565 
1566 void __init kmem_cache_init_late(void)
1567 {
1568 	struct kmem_cache *cachep;
1569 
1570 	slab_state = UP;
1571 
1572 	/* 6) resize the head arrays to their final sizes */
1573 	mutex_lock(&slab_mutex);
1574 	list_for_each_entry(cachep, &slab_caches, list)
1575 		if (enable_cpucache(cachep, GFP_NOWAIT))
1576 			BUG();
1577 	mutex_unlock(&slab_mutex);
1578 
1579 	/* Annotate slab for lockdep -- annotate the malloc caches */
1580 	init_lock_keys();
1581 
1582 	/* Done! */
1583 	slab_state = FULL;
1584 
1585 	/*
1586 	 * Register a cpu startup notifier callback that initializes
1587 	 * cpu_cache_get for all new cpus
1588 	 */
1589 	register_cpu_notifier(&cpucache_notifier);
1590 
1591 #ifdef CONFIG_NUMA
1592 	/*
1593 	 * Register a memory hotplug callback that initializes and frees
1594 	 * node.
1595 	 */
1596 	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
1597 #endif
1598 
1599 	/*
1600 	 * The reap timers are started later, with a module init call: That part
1601 	 * of the kernel is not yet operational.
1602 	 */
1603 }
1604 
1605 static int __init cpucache_init(void)
1606 {
1607 	int cpu;
1608 
1609 	/*
1610 	 * Register the timers that return unneeded pages to the page allocator
1611 	 */
1612 	for_each_online_cpu(cpu)
1613 		start_cpu_timer(cpu);
1614 
1615 	/* Done! */
1616 	slab_state = FULL;
1617 	return 0;
1618 }
1619 __initcall(cpucache_init);
1620 
1621 static noinline void
1622 slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
1623 {
1624 #if DEBUG
1625 	struct kmem_cache_node *n;
1626 	struct page *page;
1627 	unsigned long flags;
1628 	int node;
1629 	static DEFINE_RATELIMIT_STATE(slab_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
1630 				      DEFAULT_RATELIMIT_BURST);
1631 
1632 	if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slab_oom_rs))
1633 		return;
1634 
1635 	printk(KERN_WARNING
1636 		"SLAB: Unable to allocate memory on node %d (gfp=0x%x)\n",
1637 		nodeid, gfpflags);
1638 	printk(KERN_WARNING "  cache: %s, object size: %d, order: %d\n",
1639 		cachep->name, cachep->size, cachep->gfporder);
1640 
1641 	for_each_online_node(node) {
1642 		unsigned long active_objs = 0, num_objs = 0, free_objects = 0;
1643 		unsigned long active_slabs = 0, num_slabs = 0;
1644 
1645 		n = cachep->node[node];
1646 		if (!n)
1647 			continue;
1648 
1649 		spin_lock_irqsave(&n->list_lock, flags);
1650 		list_for_each_entry(page, &n->slabs_full, lru) {
1651 			active_objs += cachep->num;
1652 			active_slabs++;
1653 		}
1654 		list_for_each_entry(page, &n->slabs_partial, lru) {
1655 			active_objs += page->active;
1656 			active_slabs++;
1657 		}
1658 		list_for_each_entry(page, &n->slabs_free, lru)
1659 			num_slabs++;
1660 
1661 		free_objects += n->free_objects;
1662 		spin_unlock_irqrestore(&n->list_lock, flags);
1663 
1664 		num_slabs += active_slabs;
1665 		num_objs = num_slabs * cachep->num;
1666 		printk(KERN_WARNING
1667 			"  node %d: slabs: %ld/%ld, objs: %ld/%ld, free: %ld\n",
1668 			node, active_slabs, num_slabs, active_objs, num_objs,
1669 			free_objects);
1670 	}
1671 #endif
1672 }
1673 
1674 /*
1675  * Interface to system's page allocator. No need to hold the cache-lock.
1676  *
1677  * If we requested dmaable memory, we will get it. Even if we
1678  * did not request dmaable memory, we might get it, but that
1679  * would be relatively rare and ignorable.
1680  */
1681 static struct page *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
1682 								int nodeid)
1683 {
1684 	struct page *page;
1685 	int nr_pages;
1686 
1687 	flags |= cachep->allocflags;
1688 	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1689 		flags |= __GFP_RECLAIMABLE;
1690 
1691 	if (memcg_charge_slab(cachep, flags, cachep->gfporder))
1692 		return NULL;
1693 
1694 	page = alloc_pages_exact_node(nodeid, flags | __GFP_NOTRACK, cachep->gfporder);
1695 	if (!page) {
1696 		memcg_uncharge_slab(cachep, cachep->gfporder);
1697 		slab_out_of_memory(cachep, flags, nodeid);
1698 		return NULL;
1699 	}
1700 
1701 	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1702 	if (unlikely(page->pfmemalloc))
1703 		pfmemalloc_active = true;
1704 
1705 	nr_pages = (1 << cachep->gfporder);
1706 	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1707 		add_zone_page_state(page_zone(page),
1708 			NR_SLAB_RECLAIMABLE, nr_pages);
1709 	else
1710 		add_zone_page_state(page_zone(page),
1711 			NR_SLAB_UNRECLAIMABLE, nr_pages);
1712 	__SetPageSlab(page);
1713 	if (page->pfmemalloc)
1714 		SetPageSlabPfmemalloc(page);
1715 
1716 	if (kmemcheck_enabled && !(cachep->flags & SLAB_NOTRACK)) {
1717 		kmemcheck_alloc_shadow(page, cachep->gfporder, flags, nodeid);
1718 
1719 		if (cachep->ctor)
1720 			kmemcheck_mark_uninitialized_pages(page, nr_pages);
1721 		else
1722 			kmemcheck_mark_unallocated_pages(page, nr_pages);
1723 	}
1724 
1725 	return page;
1726 }
1727 
1728 /*
1729  * Interface to system's page release.
1730  */
1731 static void kmem_freepages(struct kmem_cache *cachep, struct page *page)
1732 {
1733 	const unsigned long nr_freed = (1 << cachep->gfporder);
1734 
1735 	kmemcheck_free_shadow(page, cachep->gfporder);
1736 
1737 	if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1738 		sub_zone_page_state(page_zone(page),
1739 				NR_SLAB_RECLAIMABLE, nr_freed);
1740 	else
1741 		sub_zone_page_state(page_zone(page),
1742 				NR_SLAB_UNRECLAIMABLE, nr_freed);
1743 
1744 	BUG_ON(!PageSlab(page));
1745 	__ClearPageSlabPfmemalloc(page);
1746 	__ClearPageSlab(page);
1747 	page_mapcount_reset(page);
1748 	page->mapping = NULL;
1749 
1750 	if (current->reclaim_state)
1751 		current->reclaim_state->reclaimed_slab += nr_freed;
1752 	__free_pages(page, cachep->gfporder);
1753 	memcg_uncharge_slab(cachep, cachep->gfporder);
1754 }
1755 
1756 static void kmem_rcu_free(struct rcu_head *head)
1757 {
1758 	struct kmem_cache *cachep;
1759 	struct page *page;
1760 
1761 	page = container_of(head, struct page, rcu_head);
1762 	cachep = page->slab_cache;
1763 
1764 	kmem_freepages(cachep, page);
1765 }
1766 
1767 #if DEBUG
1768 
1769 #ifdef CONFIG_DEBUG_PAGEALLOC
1770 static void store_stackinfo(struct kmem_cache *cachep, unsigned long *addr,
1771 			    unsigned long caller)
1772 {
1773 	int size = cachep->object_size;
1774 
1775 	addr = (unsigned long *)&((char *)addr)[obj_offset(cachep)];
1776 
1777 	if (size < 5 * sizeof(unsigned long))
1778 		return;
1779 
1780 	*addr++ = 0x12345678;
1781 	*addr++ = caller;
1782 	*addr++ = smp_processor_id();
1783 	size -= 3 * sizeof(unsigned long);
1784 	{
1785 		unsigned long *sptr = &caller;
1786 		unsigned long svalue;
1787 
1788 		while (!kstack_end(sptr)) {
1789 			svalue = *sptr++;
1790 			if (kernel_text_address(svalue)) {
1791 				*addr++ = svalue;
1792 				size -= sizeof(unsigned long);
1793 				if (size <= sizeof(unsigned long))
1794 					break;
1795 			}
1796 		}
1797 
1798 	}
1799 	*addr++ = 0x87654321;
1800 }
1801 #endif
1802 
1803 static void poison_obj(struct kmem_cache *cachep, void *addr, unsigned char val)
1804 {
1805 	int size = cachep->object_size;
1806 	addr = &((char *)addr)[obj_offset(cachep)];
1807 
1808 	memset(addr, val, size);
1809 	*(unsigned char *)(addr + size - 1) = POISON_END;
1810 }
1811 
1812 static void dump_line(char *data, int offset, int limit)
1813 {
1814 	int i;
1815 	unsigned char error = 0;
1816 	int bad_count = 0;
1817 
1818 	printk(KERN_ERR "%03x: ", offset);
1819 	for (i = 0; i < limit; i++) {
1820 		if (data[offset + i] != POISON_FREE) {
1821 			error = data[offset + i];
1822 			bad_count++;
1823 		}
1824 	}
1825 	print_hex_dump(KERN_CONT, "", 0, 16, 1,
1826 			&data[offset], limit, 1);
1827 
1828 	if (bad_count == 1) {
1829 		error ^= POISON_FREE;
1830 		if (!(error & (error - 1))) {
1831 			printk(KERN_ERR "Single bit error detected. Probably "
1832 					"bad RAM.\n");
1833 #ifdef CONFIG_X86
1834 			printk(KERN_ERR "Run memtest86+ or a similar memory "
1835 					"test tool.\n");
1836 #else
1837 			printk(KERN_ERR "Run a memory test tool.\n");
1838 #endif
1839 		}
1840 	}
1841 }
1842 #endif
1843 
1844 #if DEBUG
1845 
1846 static void print_objinfo(struct kmem_cache *cachep, void *objp, int lines)
1847 {
1848 	int i, size;
1849 	char *realobj;
1850 
1851 	if (cachep->flags & SLAB_RED_ZONE) {
1852 		printk(KERN_ERR "Redzone: 0x%llx/0x%llx.\n",
1853 			*dbg_redzone1(cachep, objp),
1854 			*dbg_redzone2(cachep, objp));
1855 	}
1856 
1857 	if (cachep->flags & SLAB_STORE_USER) {
1858 		printk(KERN_ERR "Last user: [<%p>](%pSR)\n",
1859 		       *dbg_userword(cachep, objp),
1860 		       *dbg_userword(cachep, objp));
1861 	}
1862 	realobj = (char *)objp + obj_offset(cachep);
1863 	size = cachep->object_size;
1864 	for (i = 0; i < size && lines; i += 16, lines--) {
1865 		int limit;
1866 		limit = 16;
1867 		if (i + limit > size)
1868 			limit = size - i;
1869 		dump_line(realobj, i, limit);
1870 	}
1871 }
1872 
1873 static void check_poison_obj(struct kmem_cache *cachep, void *objp)
1874 {
1875 	char *realobj;
1876 	int size, i;
1877 	int lines = 0;
1878 
1879 	realobj = (char *)objp + obj_offset(cachep);
1880 	size = cachep->object_size;
1881 
1882 	for (i = 0; i < size; i++) {
1883 		char exp = POISON_FREE;
1884 		if (i == size - 1)
1885 			exp = POISON_END;
1886 		if (realobj[i] != exp) {
1887 			int limit;
1888 			/* Mismatch ! */
1889 			/* Print header */
1890 			if (lines == 0) {
1891 				printk(KERN_ERR
1892 					"Slab corruption (%s): %s start=%p, len=%d\n",
1893 					print_tainted(), cachep->name, realobj, size);
1894 				print_objinfo(cachep, objp, 0);
1895 			}
1896 			/* Hexdump the affected line */
1897 			i = (i / 16) * 16;
1898 			limit = 16;
1899 			if (i + limit > size)
1900 				limit = size - i;
1901 			dump_line(realobj, i, limit);
1902 			i += 16;
1903 			lines++;
1904 			/* Limit to 5 lines */
1905 			if (lines > 5)
1906 				break;
1907 		}
1908 	}
1909 	if (lines != 0) {
1910 		/* Print some data about the neighboring objects, if they
1911 		 * exist:
1912 		 */
1913 		struct page *page = virt_to_head_page(objp);
1914 		unsigned int objnr;
1915 
1916 		objnr = obj_to_index(cachep, page, objp);
1917 		if (objnr) {
1918 			objp = index_to_obj(cachep, page, objnr - 1);
1919 			realobj = (char *)objp + obj_offset(cachep);
1920 			printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
1921 			       realobj, size);
1922 			print_objinfo(cachep, objp, 2);
1923 		}
1924 		if (objnr + 1 < cachep->num) {
1925 			objp = index_to_obj(cachep, page, objnr + 1);
1926 			realobj = (char *)objp + obj_offset(cachep);
1927 			printk(KERN_ERR "Next obj: start=%p, len=%d\n",
1928 			       realobj, size);
1929 			print_objinfo(cachep, objp, 2);
1930 		}
1931 	}
1932 }
1933 #endif
1934 
1935 #if DEBUG
1936 static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1937 						struct page *page)
1938 {
1939 	int i;
1940 	for (i = 0; i < cachep->num; i++) {
1941 		void *objp = index_to_obj(cachep, page, i);
1942 
1943 		if (cachep->flags & SLAB_POISON) {
1944 #ifdef CONFIG_DEBUG_PAGEALLOC
1945 			if (cachep->size % PAGE_SIZE == 0 &&
1946 					OFF_SLAB(cachep))
1947 				kernel_map_pages(virt_to_page(objp),
1948 					cachep->size / PAGE_SIZE, 1);
1949 			else
1950 				check_poison_obj(cachep, objp);
1951 #else
1952 			check_poison_obj(cachep, objp);
1953 #endif
1954 		}
1955 		if (cachep->flags & SLAB_RED_ZONE) {
1956 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1957 				slab_error(cachep, "start of a freed object "
1958 					   "was overwritten");
1959 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1960 				slab_error(cachep, "end of a freed object "
1961 					   "was overwritten");
1962 		}
1963 	}
1964 }
1965 #else
1966 static void slab_destroy_debugcheck(struct kmem_cache *cachep,
1967 						struct page *page)
1968 {
1969 }
1970 #endif
1971 
1972 /**
1973  * slab_destroy - destroy and release all objects in a slab
1974  * @cachep: cache pointer being destroyed
1975  * @page: page pointer being destroyed
1976  *
1977  * Destroy all the objs in a slab, and release the mem back to the system.
1978  * Before calling the slab must have been unlinked from the cache.  The
1979  * cache-lock is not held/needed.
1980  */
1981 static void slab_destroy(struct kmem_cache *cachep, struct page *page)
1982 {
1983 	void *freelist;
1984 
1985 	freelist = page->freelist;
1986 	slab_destroy_debugcheck(cachep, page);
1987 	if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
1988 		struct rcu_head *head;
1989 
1990 		/*
1991 		 * RCU free overloads the RCU head over the LRU.
1992 		 * slab_page has been overloeaded over the LRU,
1993 		 * however it is not used from now on so that
1994 		 * we can use it safely.
1995 		 */
1996 		head = (void *)&page->rcu_head;
1997 		call_rcu(head, kmem_rcu_free);
1998 
1999 	} else {
2000 		kmem_freepages(cachep, page);
2001 	}
2002 
2003 	/*
2004 	 * From now on, we don't use freelist
2005 	 * although actual page can be freed in rcu context
2006 	 */
2007 	if (OFF_SLAB(cachep))
2008 		kmem_cache_free(cachep->freelist_cache, freelist);
2009 }
2010 
2011 /**
2012  * calculate_slab_order - calculate size (page order) of slabs
2013  * @cachep: pointer to the cache that is being created
2014  * @size: size of objects to be created in this cache.
2015  * @align: required alignment for the objects.
2016  * @flags: slab allocation flags
2017  *
2018  * Also calculates the number of objects per slab.
2019  *
2020  * This could be made much more intelligent.  For now, try to avoid using
2021  * high order pages for slabs.  When the gfp() functions are more friendly
2022  * towards high-order requests, this should be changed.
2023  */
2024 static size_t calculate_slab_order(struct kmem_cache *cachep,
2025 			size_t size, size_t align, unsigned long flags)
2026 {
2027 	unsigned long offslab_limit;
2028 	size_t left_over = 0;
2029 	int gfporder;
2030 
2031 	for (gfporder = 0; gfporder <= KMALLOC_MAX_ORDER; gfporder++) {
2032 		unsigned int num;
2033 		size_t remainder;
2034 
2035 		cache_estimate(gfporder, size, align, flags, &remainder, &num);
2036 		if (!num)
2037 			continue;
2038 
2039 		/* Can't handle number of objects more than SLAB_OBJ_MAX_NUM */
2040 		if (num > SLAB_OBJ_MAX_NUM)
2041 			break;
2042 
2043 		if (flags & CFLGS_OFF_SLAB) {
2044 			/*
2045 			 * Max number of objs-per-slab for caches which
2046 			 * use off-slab slabs. Needed to avoid a possible
2047 			 * looping condition in cache_grow().
2048 			 */
2049 			offslab_limit = size;
2050 			offslab_limit /= sizeof(freelist_idx_t);
2051 
2052  			if (num > offslab_limit)
2053 				break;
2054 		}
2055 
2056 		/* Found something acceptable - save it away */
2057 		cachep->num = num;
2058 		cachep->gfporder = gfporder;
2059 		left_over = remainder;
2060 
2061 		/*
2062 		 * A VFS-reclaimable slab tends to have most allocations
2063 		 * as GFP_NOFS and we really don't want to have to be allocating
2064 		 * higher-order pages when we are unable to shrink dcache.
2065 		 */
2066 		if (flags & SLAB_RECLAIM_ACCOUNT)
2067 			break;
2068 
2069 		/*
2070 		 * Large number of objects is good, but very large slabs are
2071 		 * currently bad for the gfp()s.
2072 		 */
2073 		if (gfporder >= slab_max_order)
2074 			break;
2075 
2076 		/*
2077 		 * Acceptable internal fragmentation?
2078 		 */
2079 		if (left_over * 8 <= (PAGE_SIZE << gfporder))
2080 			break;
2081 	}
2082 	return left_over;
2083 }
2084 
2085 static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2086 {
2087 	if (slab_state >= FULL)
2088 		return enable_cpucache(cachep, gfp);
2089 
2090 	if (slab_state == DOWN) {
2091 		/*
2092 		 * Note: Creation of first cache (kmem_cache).
2093 		 * The setup_node is taken care
2094 		 * of by the caller of __kmem_cache_create
2095 		 */
2096 		cachep->array[smp_processor_id()] = &initarray_generic.cache;
2097 		slab_state = PARTIAL;
2098 	} else if (slab_state == PARTIAL) {
2099 		/*
2100 		 * Note: the second kmem_cache_create must create the cache
2101 		 * that's used by kmalloc(24), otherwise the creation of
2102 		 * further caches will BUG().
2103 		 */
2104 		cachep->array[smp_processor_id()] = &initarray_generic.cache;
2105 
2106 		/*
2107 		 * If the cache that's used by kmalloc(sizeof(kmem_cache_node)) is
2108 		 * the second cache, then we need to set up all its node/,
2109 		 * otherwise the creation of further caches will BUG().
2110 		 */
2111 		set_up_node(cachep, SIZE_AC);
2112 		if (INDEX_AC == INDEX_NODE)
2113 			slab_state = PARTIAL_NODE;
2114 		else
2115 			slab_state = PARTIAL_ARRAYCACHE;
2116 	} else {
2117 		/* Remaining boot caches */
2118 		cachep->array[smp_processor_id()] =
2119 			kmalloc(sizeof(struct arraycache_init), gfp);
2120 
2121 		if (slab_state == PARTIAL_ARRAYCACHE) {
2122 			set_up_node(cachep, SIZE_NODE);
2123 			slab_state = PARTIAL_NODE;
2124 		} else {
2125 			int node;
2126 			for_each_online_node(node) {
2127 				cachep->node[node] =
2128 				    kmalloc_node(sizeof(struct kmem_cache_node),
2129 						gfp, node);
2130 				BUG_ON(!cachep->node[node]);
2131 				kmem_cache_node_init(cachep->node[node]);
2132 			}
2133 		}
2134 	}
2135 	cachep->node[numa_mem_id()]->next_reap =
2136 			jiffies + REAPTIMEOUT_NODE +
2137 			((unsigned long)cachep) % REAPTIMEOUT_NODE;
2138 
2139 	cpu_cache_get(cachep)->avail = 0;
2140 	cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES;
2141 	cpu_cache_get(cachep)->batchcount = 1;
2142 	cpu_cache_get(cachep)->touched = 0;
2143 	cachep->batchcount = 1;
2144 	cachep->limit = BOOT_CPUCACHE_ENTRIES;
2145 	return 0;
2146 }
2147 
2148 /**
2149  * __kmem_cache_create - Create a cache.
2150  * @cachep: cache management descriptor
2151  * @flags: SLAB flags
2152  *
2153  * Returns a ptr to the cache on success, NULL on failure.
2154  * Cannot be called within a int, but can be interrupted.
2155  * The @ctor is run when new pages are allocated by the cache.
2156  *
2157  * The flags are
2158  *
2159  * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5)
2160  * to catch references to uninitialised memory.
2161  *
2162  * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check
2163  * for buffer overruns.
2164  *
2165  * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
2166  * cacheline.  This can be beneficial if you're counting cycles as closely
2167  * as davem.
2168  */
2169 int
2170 __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2171 {
2172 	size_t left_over, freelist_size, ralign;
2173 	gfp_t gfp;
2174 	int err;
2175 	size_t size = cachep->size;
2176 
2177 #if DEBUG
2178 #if FORCED_DEBUG
2179 	/*
2180 	 * Enable redzoning and last user accounting, except for caches with
2181 	 * large objects, if the increased size would increase the object size
2182 	 * above the next power of two: caches with object sizes just above a
2183 	 * power of two have a significant amount of internal fragmentation.
2184 	 */
2185 	if (size < 4096 || fls(size - 1) == fls(size-1 + REDZONE_ALIGN +
2186 						2 * sizeof(unsigned long long)))
2187 		flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
2188 	if (!(flags & SLAB_DESTROY_BY_RCU))
2189 		flags |= SLAB_POISON;
2190 #endif
2191 	if (flags & SLAB_DESTROY_BY_RCU)
2192 		BUG_ON(flags & SLAB_POISON);
2193 #endif
2194 
2195 	/*
2196 	 * Check that size is in terms of words.  This is needed to avoid
2197 	 * unaligned accesses for some archs when redzoning is used, and makes
2198 	 * sure any on-slab bufctl's are also correctly aligned.
2199 	 */
2200 	if (size & (BYTES_PER_WORD - 1)) {
2201 		size += (BYTES_PER_WORD - 1);
2202 		size &= ~(BYTES_PER_WORD - 1);
2203 	}
2204 
2205 	/*
2206 	 * Redzoning and user store require word alignment or possibly larger.
2207 	 * Note this will be overridden by architecture or caller mandated
2208 	 * alignment if either is greater than BYTES_PER_WORD.
2209 	 */
2210 	if (flags & SLAB_STORE_USER)
2211 		ralign = BYTES_PER_WORD;
2212 
2213 	if (flags & SLAB_RED_ZONE) {
2214 		ralign = REDZONE_ALIGN;
2215 		/* If redzoning, ensure that the second redzone is suitably
2216 		 * aligned, by adjusting the object size accordingly. */
2217 		size += REDZONE_ALIGN - 1;
2218 		size &= ~(REDZONE_ALIGN - 1);
2219 	}
2220 
2221 	/* 3) caller mandated alignment */
2222 	if (ralign < cachep->align) {
2223 		ralign = cachep->align;
2224 	}
2225 	/* disable debug if necessary */
2226 	if (ralign > __alignof__(unsigned long long))
2227 		flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2228 	/*
2229 	 * 4) Store it.
2230 	 */
2231 	cachep->align = ralign;
2232 
2233 	if (slab_is_available())
2234 		gfp = GFP_KERNEL;
2235 	else
2236 		gfp = GFP_NOWAIT;
2237 
2238 	setup_node_pointer(cachep);
2239 #if DEBUG
2240 
2241 	/*
2242 	 * Both debugging options require word-alignment which is calculated
2243 	 * into align above.
2244 	 */
2245 	if (flags & SLAB_RED_ZONE) {
2246 		/* add space for red zone words */
2247 		cachep->obj_offset += sizeof(unsigned long long);
2248 		size += 2 * sizeof(unsigned long long);
2249 	}
2250 	if (flags & SLAB_STORE_USER) {
2251 		/* user store requires one word storage behind the end of
2252 		 * the real object. But if the second red zone needs to be
2253 		 * aligned to 64 bits, we must allow that much space.
2254 		 */
2255 		if (flags & SLAB_RED_ZONE)
2256 			size += REDZONE_ALIGN;
2257 		else
2258 			size += BYTES_PER_WORD;
2259 	}
2260 #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
2261 	if (size >= kmalloc_size(INDEX_NODE + 1)
2262 	    && cachep->object_size > cache_line_size()
2263 	    && ALIGN(size, cachep->align) < PAGE_SIZE) {
2264 		cachep->obj_offset += PAGE_SIZE - ALIGN(size, cachep->align);
2265 		size = PAGE_SIZE;
2266 	}
2267 #endif
2268 #endif
2269 
2270 	/*
2271 	 * Determine if the slab management is 'on' or 'off' slab.
2272 	 * (bootstrapping cannot cope with offslab caches so don't do
2273 	 * it too early on. Always use on-slab management when
2274 	 * SLAB_NOLEAKTRACE to avoid recursive calls into kmemleak)
2275 	 */
2276 	if ((size >= (PAGE_SIZE >> 5)) && !slab_early_init &&
2277 	    !(flags & SLAB_NOLEAKTRACE))
2278 		/*
2279 		 * Size is large, assume best to place the slab management obj
2280 		 * off-slab (should allow better packing of objs).
2281 		 */
2282 		flags |= CFLGS_OFF_SLAB;
2283 
2284 	size = ALIGN(size, cachep->align);
2285 	/*
2286 	 * We should restrict the number of objects in a slab to implement
2287 	 * byte sized index. Refer comment on SLAB_OBJ_MIN_SIZE definition.
2288 	 */
2289 	if (FREELIST_BYTE_INDEX && size < SLAB_OBJ_MIN_SIZE)
2290 		size = ALIGN(SLAB_OBJ_MIN_SIZE, cachep->align);
2291 
2292 	left_over = calculate_slab_order(cachep, size, cachep->align, flags);
2293 
2294 	if (!cachep->num)
2295 		return -E2BIG;
2296 
2297 	freelist_size =
2298 		ALIGN(cachep->num * sizeof(freelist_idx_t), cachep->align);
2299 
2300 	/*
2301 	 * If the slab has been placed off-slab, and we have enough space then
2302 	 * move it on-slab. This is at the expense of any extra colouring.
2303 	 */
2304 	if (flags & CFLGS_OFF_SLAB && left_over >= freelist_size) {
2305 		flags &= ~CFLGS_OFF_SLAB;
2306 		left_over -= freelist_size;
2307 	}
2308 
2309 	if (flags & CFLGS_OFF_SLAB) {
2310 		/* really off slab. No need for manual alignment */
2311 		freelist_size = cachep->num * sizeof(freelist_idx_t);
2312 
2313 #ifdef CONFIG_PAGE_POISONING
2314 		/* If we're going to use the generic kernel_map_pages()
2315 		 * poisoning, then it's going to smash the contents of
2316 		 * the redzone and userword anyhow, so switch them off.
2317 		 */
2318 		if (size % PAGE_SIZE == 0 && flags & SLAB_POISON)
2319 			flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
2320 #endif
2321 	}
2322 
2323 	cachep->colour_off = cache_line_size();
2324 	/* Offset must be a multiple of the alignment. */
2325 	if (cachep->colour_off < cachep->align)
2326 		cachep->colour_off = cachep->align;
2327 	cachep->colour = left_over / cachep->colour_off;
2328 	cachep->freelist_size = freelist_size;
2329 	cachep->flags = flags;
2330 	cachep->allocflags = __GFP_COMP;
2331 	if (CONFIG_ZONE_DMA_FLAG && (flags & SLAB_CACHE_DMA))
2332 		cachep->allocflags |= GFP_DMA;
2333 	cachep->size = size;
2334 	cachep->reciprocal_buffer_size = reciprocal_value(size);
2335 
2336 	if (flags & CFLGS_OFF_SLAB) {
2337 		cachep->freelist_cache = kmalloc_slab(freelist_size, 0u);
2338 		/*
2339 		 * This is a possibility for one of the kmalloc_{dma,}_caches.
2340 		 * But since we go off slab only for object size greater than
2341 		 * PAGE_SIZE/8, and kmalloc_{dma,}_caches get created
2342 		 * in ascending order,this should not happen at all.
2343 		 * But leave a BUG_ON for some lucky dude.
2344 		 */
2345 		BUG_ON(ZERO_OR_NULL_PTR(cachep->freelist_cache));
2346 	}
2347 
2348 	err = setup_cpu_cache(cachep, gfp);
2349 	if (err) {
2350 		__kmem_cache_shutdown(cachep);
2351 		return err;
2352 	}
2353 
2354 	if (flags & SLAB_DEBUG_OBJECTS) {
2355 		/*
2356 		 * Would deadlock through slab_destroy()->call_rcu()->
2357 		 * debug_object_activate()->kmem_cache_alloc().
2358 		 */
2359 		WARN_ON_ONCE(flags & SLAB_DESTROY_BY_RCU);
2360 
2361 		slab_set_debugobj_lock_classes(cachep);
2362 	} else if (!OFF_SLAB(cachep) && !(flags & SLAB_DESTROY_BY_RCU))
2363 		on_slab_lock_classes(cachep);
2364 
2365 	return 0;
2366 }
2367 
2368 #if DEBUG
2369 static void check_irq_off(void)
2370 {
2371 	BUG_ON(!irqs_disabled());
2372 }
2373 
2374 static void check_irq_on(void)
2375 {
2376 	BUG_ON(irqs_disabled());
2377 }
2378 
2379 static void check_spinlock_acquired(struct kmem_cache *cachep)
2380 {
2381 #ifdef CONFIG_SMP
2382 	check_irq_off();
2383 	assert_spin_locked(&cachep->node[numa_mem_id()]->list_lock);
2384 #endif
2385 }
2386 
2387 static void check_spinlock_acquired_node(struct kmem_cache *cachep, int node)
2388 {
2389 #ifdef CONFIG_SMP
2390 	check_irq_off();
2391 	assert_spin_locked(&cachep->node[node]->list_lock);
2392 #endif
2393 }
2394 
2395 #else
2396 #define check_irq_off()	do { } while(0)
2397 #define check_irq_on()	do { } while(0)
2398 #define check_spinlock_acquired(x) do { } while(0)
2399 #define check_spinlock_acquired_node(x, y) do { } while(0)
2400 #endif
2401 
2402 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
2403 			struct array_cache *ac,
2404 			int force, int node);
2405 
2406 static void do_drain(void *arg)
2407 {
2408 	struct kmem_cache *cachep = arg;
2409 	struct array_cache *ac;
2410 	int node = numa_mem_id();
2411 
2412 	check_irq_off();
2413 	ac = cpu_cache_get(cachep);
2414 	spin_lock(&cachep->node[node]->list_lock);
2415 	free_block(cachep, ac->entry, ac->avail, node);
2416 	spin_unlock(&cachep->node[node]->list_lock);
2417 	ac->avail = 0;
2418 }
2419 
2420 static void drain_cpu_caches(struct kmem_cache *cachep)
2421 {
2422 	struct kmem_cache_node *n;
2423 	int node;
2424 
2425 	on_each_cpu(do_drain, cachep, 1);
2426 	check_irq_on();
2427 	for_each_online_node(node) {
2428 		n = cachep->node[node];
2429 		if (n && n->alien)
2430 			drain_alien_cache(cachep, n->alien);
2431 	}
2432 
2433 	for_each_online_node(node) {
2434 		n = cachep->node[node];
2435 		if (n)
2436 			drain_array(cachep, n, n->shared, 1, node);
2437 	}
2438 }
2439 
2440 /*
2441  * Remove slabs from the list of free slabs.
2442  * Specify the number of slabs to drain in tofree.
2443  *
2444  * Returns the actual number of slabs released.
2445  */
2446 static int drain_freelist(struct kmem_cache *cache,
2447 			struct kmem_cache_node *n, int tofree)
2448 {
2449 	struct list_head *p;
2450 	int nr_freed;
2451 	struct page *page;
2452 
2453 	nr_freed = 0;
2454 	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
2455 
2456 		spin_lock_irq(&n->list_lock);
2457 		p = n->slabs_free.prev;
2458 		if (p == &n->slabs_free) {
2459 			spin_unlock_irq(&n->list_lock);
2460 			goto out;
2461 		}
2462 
2463 		page = list_entry(p, struct page, lru);
2464 #if DEBUG
2465 		BUG_ON(page->active);
2466 #endif
2467 		list_del(&page->lru);
2468 		/*
2469 		 * Safe to drop the lock. The slab is no longer linked
2470 		 * to the cache.
2471 		 */
2472 		n->free_objects -= cache->num;
2473 		spin_unlock_irq(&n->list_lock);
2474 		slab_destroy(cache, page);
2475 		nr_freed++;
2476 	}
2477 out:
2478 	return nr_freed;
2479 }
2480 
2481 int __kmem_cache_shrink(struct kmem_cache *cachep)
2482 {
2483 	int ret = 0, i = 0;
2484 	struct kmem_cache_node *n;
2485 
2486 	drain_cpu_caches(cachep);
2487 
2488 	check_irq_on();
2489 	for_each_online_node(i) {
2490 		n = cachep->node[i];
2491 		if (!n)
2492 			continue;
2493 
2494 		drain_freelist(cachep, n, slabs_tofree(cachep, n));
2495 
2496 		ret += !list_empty(&n->slabs_full) ||
2497 			!list_empty(&n->slabs_partial);
2498 	}
2499 	return (ret ? 1 : 0);
2500 }
2501 
2502 int __kmem_cache_shutdown(struct kmem_cache *cachep)
2503 {
2504 	int i;
2505 	struct kmem_cache_node *n;
2506 	int rc = __kmem_cache_shrink(cachep);
2507 
2508 	if (rc)
2509 		return rc;
2510 
2511 	for_each_online_cpu(i)
2512 	    kfree(cachep->array[i]);
2513 
2514 	/* NUMA: free the node structures */
2515 	for_each_online_node(i) {
2516 		n = cachep->node[i];
2517 		if (n) {
2518 			kfree(n->shared);
2519 			free_alien_cache(n->alien);
2520 			kfree(n);
2521 		}
2522 	}
2523 	return 0;
2524 }
2525 
2526 /*
2527  * Get the memory for a slab management obj.
2528  *
2529  * For a slab cache when the slab descriptor is off-slab, the
2530  * slab descriptor can't come from the same cache which is being created,
2531  * Because if it is the case, that means we defer the creation of
2532  * the kmalloc_{dma,}_cache of size sizeof(slab descriptor) to this point.
2533  * And we eventually call down to __kmem_cache_create(), which
2534  * in turn looks up in the kmalloc_{dma,}_caches for the disired-size one.
2535  * This is a "chicken-and-egg" problem.
2536  *
2537  * So the off-slab slab descriptor shall come from the kmalloc_{dma,}_caches,
2538  * which are all initialized during kmem_cache_init().
2539  */
2540 static void *alloc_slabmgmt(struct kmem_cache *cachep,
2541 				   struct page *page, int colour_off,
2542 				   gfp_t local_flags, int nodeid)
2543 {
2544 	void *freelist;
2545 	void *addr = page_address(page);
2546 
2547 	if (OFF_SLAB(cachep)) {
2548 		/* Slab management obj is off-slab. */
2549 		freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2550 					      local_flags, nodeid);
2551 		if (!freelist)
2552 			return NULL;
2553 	} else {
2554 		freelist = addr + colour_off;
2555 		colour_off += cachep->freelist_size;
2556 	}
2557 	page->active = 0;
2558 	page->s_mem = addr + colour_off;
2559 	return freelist;
2560 }
2561 
2562 static inline freelist_idx_t get_free_obj(struct page *page, unsigned int idx)
2563 {
2564 	return ((freelist_idx_t *)page->freelist)[idx];
2565 }
2566 
2567 static inline void set_free_obj(struct page *page,
2568 					unsigned int idx, freelist_idx_t val)
2569 {
2570 	((freelist_idx_t *)(page->freelist))[idx] = val;
2571 }
2572 
2573 static void cache_init_objs(struct kmem_cache *cachep,
2574 			    struct page *page)
2575 {
2576 	int i;
2577 
2578 	for (i = 0; i < cachep->num; i++) {
2579 		void *objp = index_to_obj(cachep, page, i);
2580 #if DEBUG
2581 		/* need to poison the objs? */
2582 		if (cachep->flags & SLAB_POISON)
2583 			poison_obj(cachep, objp, POISON_FREE);
2584 		if (cachep->flags & SLAB_STORE_USER)
2585 			*dbg_userword(cachep, objp) = NULL;
2586 
2587 		if (cachep->flags & SLAB_RED_ZONE) {
2588 			*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2589 			*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2590 		}
2591 		/*
2592 		 * Constructors are not allowed to allocate memory from the same
2593 		 * cache which they are a constructor for.  Otherwise, deadlock.
2594 		 * They must also be threaded.
2595 		 */
2596 		if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2597 			cachep->ctor(objp + obj_offset(cachep));
2598 
2599 		if (cachep->flags & SLAB_RED_ZONE) {
2600 			if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2601 				slab_error(cachep, "constructor overwrote the"
2602 					   " end of an object");
2603 			if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2604 				slab_error(cachep, "constructor overwrote the"
2605 					   " start of an object");
2606 		}
2607 		if ((cachep->size % PAGE_SIZE) == 0 &&
2608 			    OFF_SLAB(cachep) && cachep->flags & SLAB_POISON)
2609 			kernel_map_pages(virt_to_page(objp),
2610 					 cachep->size / PAGE_SIZE, 0);
2611 #else
2612 		if (cachep->ctor)
2613 			cachep->ctor(objp);
2614 #endif
2615 		set_free_obj(page, i, i);
2616 	}
2617 }
2618 
2619 static void kmem_flagcheck(struct kmem_cache *cachep, gfp_t flags)
2620 {
2621 	if (CONFIG_ZONE_DMA_FLAG) {
2622 		if (flags & GFP_DMA)
2623 			BUG_ON(!(cachep->allocflags & GFP_DMA));
2624 		else
2625 			BUG_ON(cachep->allocflags & GFP_DMA);
2626 	}
2627 }
2628 
2629 static void *slab_get_obj(struct kmem_cache *cachep, struct page *page,
2630 				int nodeid)
2631 {
2632 	void *objp;
2633 
2634 	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2635 	page->active++;
2636 #if DEBUG
2637 	WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
2638 #endif
2639 
2640 	return objp;
2641 }
2642 
2643 static void slab_put_obj(struct kmem_cache *cachep, struct page *page,
2644 				void *objp, int nodeid)
2645 {
2646 	unsigned int objnr = obj_to_index(cachep, page, objp);
2647 #if DEBUG
2648 	unsigned int i;
2649 
2650 	/* Verify that the slab belongs to the intended node */
2651 	WARN_ON(page_to_nid(virt_to_page(objp)) != nodeid);
2652 
2653 	/* Verify double free bug */
2654 	for (i = page->active; i < cachep->num; i++) {
2655 		if (get_free_obj(page, i) == objnr) {
2656 			printk(KERN_ERR "slab: double free detected in cache "
2657 					"'%s', objp %p\n", cachep->name, objp);
2658 			BUG();
2659 		}
2660 	}
2661 #endif
2662 	page->active--;
2663 	set_free_obj(page, page->active, objnr);
2664 }
2665 
2666 /*
2667  * Map pages beginning at addr to the given cache and slab. This is required
2668  * for the slab allocator to be able to lookup the cache and slab of a
2669  * virtual address for kfree, ksize, and slab debugging.
2670  */
2671 static void slab_map_pages(struct kmem_cache *cache, struct page *page,
2672 			   void *freelist)
2673 {
2674 	page->slab_cache = cache;
2675 	page->freelist = freelist;
2676 }
2677 
2678 /*
2679  * Grow (by 1) the number of slabs within a cache.  This is called by
2680  * kmem_cache_alloc() when there are no active objs left in a cache.
2681  */
2682 static int cache_grow(struct kmem_cache *cachep,
2683 		gfp_t flags, int nodeid, struct page *page)
2684 {
2685 	void *freelist;
2686 	size_t offset;
2687 	gfp_t local_flags;
2688 	struct kmem_cache_node *n;
2689 
2690 	/*
2691 	 * Be lazy and only check for valid flags here,  keeping it out of the
2692 	 * critical path in kmem_cache_alloc().
2693 	 */
2694 	BUG_ON(flags & GFP_SLAB_BUG_MASK);
2695 	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
2696 
2697 	/* Take the node list lock to change the colour_next on this node */
2698 	check_irq_off();
2699 	n = cachep->node[nodeid];
2700 	spin_lock(&n->list_lock);
2701 
2702 	/* Get colour for the slab, and cal the next value. */
2703 	offset = n->colour_next;
2704 	n->colour_next++;
2705 	if (n->colour_next >= cachep->colour)
2706 		n->colour_next = 0;
2707 	spin_unlock(&n->list_lock);
2708 
2709 	offset *= cachep->colour_off;
2710 
2711 	if (local_flags & __GFP_WAIT)
2712 		local_irq_enable();
2713 
2714 	/*
2715 	 * The test for missing atomic flag is performed here, rather than
2716 	 * the more obvious place, simply to reduce the critical path length
2717 	 * in kmem_cache_alloc(). If a caller is seriously mis-behaving they
2718 	 * will eventually be caught here (where it matters).
2719 	 */
2720 	kmem_flagcheck(cachep, flags);
2721 
2722 	/*
2723 	 * Get mem for the objs.  Attempt to allocate a physical page from
2724 	 * 'nodeid'.
2725 	 */
2726 	if (!page)
2727 		page = kmem_getpages(cachep, local_flags, nodeid);
2728 	if (!page)
2729 		goto failed;
2730 
2731 	/* Get slab management. */
2732 	freelist = alloc_slabmgmt(cachep, page, offset,
2733 			local_flags & ~GFP_CONSTRAINT_MASK, nodeid);
2734 	if (!freelist)
2735 		goto opps1;
2736 
2737 	slab_map_pages(cachep, page, freelist);
2738 
2739 	cache_init_objs(cachep, page);
2740 
2741 	if (local_flags & __GFP_WAIT)
2742 		local_irq_disable();
2743 	check_irq_off();
2744 	spin_lock(&n->list_lock);
2745 
2746 	/* Make slab active. */
2747 	list_add_tail(&page->lru, &(n->slabs_free));
2748 	STATS_INC_GROWN(cachep);
2749 	n->free_objects += cachep->num;
2750 	spin_unlock(&n->list_lock);
2751 	return 1;
2752 opps1:
2753 	kmem_freepages(cachep, page);
2754 failed:
2755 	if (local_flags & __GFP_WAIT)
2756 		local_irq_disable();
2757 	return 0;
2758 }
2759 
2760 #if DEBUG
2761 
2762 /*
2763  * Perform extra freeing checks:
2764  * - detect bad pointers.
2765  * - POISON/RED_ZONE checking
2766  */
2767 static void kfree_debugcheck(const void *objp)
2768 {
2769 	if (!virt_addr_valid(objp)) {
2770 		printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
2771 		       (unsigned long)objp);
2772 		BUG();
2773 	}
2774 }
2775 
2776 static inline void verify_redzone_free(struct kmem_cache *cache, void *obj)
2777 {
2778 	unsigned long long redzone1, redzone2;
2779 
2780 	redzone1 = *dbg_redzone1(cache, obj);
2781 	redzone2 = *dbg_redzone2(cache, obj);
2782 
2783 	/*
2784 	 * Redzone is ok.
2785 	 */
2786 	if (redzone1 == RED_ACTIVE && redzone2 == RED_ACTIVE)
2787 		return;
2788 
2789 	if (redzone1 == RED_INACTIVE && redzone2 == RED_INACTIVE)
2790 		slab_error(cache, "double free detected");
2791 	else
2792 		slab_error(cache, "memory outside object was overwritten");
2793 
2794 	printk(KERN_ERR "%p: redzone 1:0x%llx, redzone 2:0x%llx.\n",
2795 			obj, redzone1, redzone2);
2796 }
2797 
2798 static void *cache_free_debugcheck(struct kmem_cache *cachep, void *objp,
2799 				   unsigned long caller)
2800 {
2801 	unsigned int objnr;
2802 	struct page *page;
2803 
2804 	BUG_ON(virt_to_cache(objp) != cachep);
2805 
2806 	objp -= obj_offset(cachep);
2807 	kfree_debugcheck(objp);
2808 	page = virt_to_head_page(objp);
2809 
2810 	if (cachep->flags & SLAB_RED_ZONE) {
2811 		verify_redzone_free(cachep, objp);
2812 		*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2813 		*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2814 	}
2815 	if (cachep->flags & SLAB_STORE_USER)
2816 		*dbg_userword(cachep, objp) = (void *)caller;
2817 
2818 	objnr = obj_to_index(cachep, page, objp);
2819 
2820 	BUG_ON(objnr >= cachep->num);
2821 	BUG_ON(objp != index_to_obj(cachep, page, objnr));
2822 
2823 	if (cachep->flags & SLAB_POISON) {
2824 #ifdef CONFIG_DEBUG_PAGEALLOC
2825 		if ((cachep->size % PAGE_SIZE)==0 && OFF_SLAB(cachep)) {
2826 			store_stackinfo(cachep, objp, caller);
2827 			kernel_map_pages(virt_to_page(objp),
2828 					 cachep->size / PAGE_SIZE, 0);
2829 		} else {
2830 			poison_obj(cachep, objp, POISON_FREE);
2831 		}
2832 #else
2833 		poison_obj(cachep, objp, POISON_FREE);
2834 #endif
2835 	}
2836 	return objp;
2837 }
2838 
2839 #else
2840 #define kfree_debugcheck(x) do { } while(0)
2841 #define cache_free_debugcheck(x,objp,z) (objp)
2842 #endif
2843 
2844 static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags,
2845 							bool force_refill)
2846 {
2847 	int batchcount;
2848 	struct kmem_cache_node *n;
2849 	struct array_cache *ac;
2850 	int node;
2851 
2852 	check_irq_off();
2853 	node = numa_mem_id();
2854 	if (unlikely(force_refill))
2855 		goto force_grow;
2856 retry:
2857 	ac = cpu_cache_get(cachep);
2858 	batchcount = ac->batchcount;
2859 	if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
2860 		/*
2861 		 * If there was little recent activity on this cache, then
2862 		 * perform only a partial refill.  Otherwise we could generate
2863 		 * refill bouncing.
2864 		 */
2865 		batchcount = BATCHREFILL_LIMIT;
2866 	}
2867 	n = cachep->node[node];
2868 
2869 	BUG_ON(ac->avail > 0 || !n);
2870 	spin_lock(&n->list_lock);
2871 
2872 	/* See if we can refill from the shared array */
2873 	if (n->shared && transfer_objects(ac, n->shared, batchcount)) {
2874 		n->shared->touched = 1;
2875 		goto alloc_done;
2876 	}
2877 
2878 	while (batchcount > 0) {
2879 		struct list_head *entry;
2880 		struct page *page;
2881 		/* Get slab alloc is to come from. */
2882 		entry = n->slabs_partial.next;
2883 		if (entry == &n->slabs_partial) {
2884 			n->free_touched = 1;
2885 			entry = n->slabs_free.next;
2886 			if (entry == &n->slabs_free)
2887 				goto must_grow;
2888 		}
2889 
2890 		page = list_entry(entry, struct page, lru);
2891 		check_spinlock_acquired(cachep);
2892 
2893 		/*
2894 		 * The slab was either on partial or free list so
2895 		 * there must be at least one object available for
2896 		 * allocation.
2897 		 */
2898 		BUG_ON(page->active >= cachep->num);
2899 
2900 		while (page->active < cachep->num && batchcount--) {
2901 			STATS_INC_ALLOCED(cachep);
2902 			STATS_INC_ACTIVE(cachep);
2903 			STATS_SET_HIGH(cachep);
2904 
2905 			ac_put_obj(cachep, ac, slab_get_obj(cachep, page,
2906 									node));
2907 		}
2908 
2909 		/* move slabp to correct slabp list: */
2910 		list_del(&page->lru);
2911 		if (page->active == cachep->num)
2912 			list_add(&page->lru, &n->slabs_full);
2913 		else
2914 			list_add(&page->lru, &n->slabs_partial);
2915 	}
2916 
2917 must_grow:
2918 	n->free_objects -= ac->avail;
2919 alloc_done:
2920 	spin_unlock(&n->list_lock);
2921 
2922 	if (unlikely(!ac->avail)) {
2923 		int x;
2924 force_grow:
2925 		x = cache_grow(cachep, flags | GFP_THISNODE, node, NULL);
2926 
2927 		/* cache_grow can reenable interrupts, then ac could change. */
2928 		ac = cpu_cache_get(cachep);
2929 		node = numa_mem_id();
2930 
2931 		/* no objects in sight? abort */
2932 		if (!x && (ac->avail == 0 || force_refill))
2933 			return NULL;
2934 
2935 		if (!ac->avail)		/* objects refilled by interrupt? */
2936 			goto retry;
2937 	}
2938 	ac->touched = 1;
2939 
2940 	return ac_get_obj(cachep, ac, flags, force_refill);
2941 }
2942 
2943 static inline void cache_alloc_debugcheck_before(struct kmem_cache *cachep,
2944 						gfp_t flags)
2945 {
2946 	might_sleep_if(flags & __GFP_WAIT);
2947 #if DEBUG
2948 	kmem_flagcheck(cachep, flags);
2949 #endif
2950 }
2951 
2952 #if DEBUG
2953 static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
2954 				gfp_t flags, void *objp, unsigned long caller)
2955 {
2956 	if (!objp)
2957 		return objp;
2958 	if (cachep->flags & SLAB_POISON) {
2959 #ifdef CONFIG_DEBUG_PAGEALLOC
2960 		if ((cachep->size % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
2961 			kernel_map_pages(virt_to_page(objp),
2962 					 cachep->size / PAGE_SIZE, 1);
2963 		else
2964 			check_poison_obj(cachep, objp);
2965 #else
2966 		check_poison_obj(cachep, objp);
2967 #endif
2968 		poison_obj(cachep, objp, POISON_INUSE);
2969 	}
2970 	if (cachep->flags & SLAB_STORE_USER)
2971 		*dbg_userword(cachep, objp) = (void *)caller;
2972 
2973 	if (cachep->flags & SLAB_RED_ZONE) {
2974 		if (*dbg_redzone1(cachep, objp) != RED_INACTIVE ||
2975 				*dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2976 			slab_error(cachep, "double free, or memory outside"
2977 						" object was overwritten");
2978 			printk(KERN_ERR
2979 				"%p: redzone 1:0x%llx, redzone 2:0x%llx\n",
2980 				objp, *dbg_redzone1(cachep, objp),
2981 				*dbg_redzone2(cachep, objp));
2982 		}
2983 		*dbg_redzone1(cachep, objp) = RED_ACTIVE;
2984 		*dbg_redzone2(cachep, objp) = RED_ACTIVE;
2985 	}
2986 	objp += obj_offset(cachep);
2987 	if (cachep->ctor && cachep->flags & SLAB_POISON)
2988 		cachep->ctor(objp);
2989 	if (ARCH_SLAB_MINALIGN &&
2990 	    ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
2991 		printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n",
2992 		       objp, (int)ARCH_SLAB_MINALIGN);
2993 	}
2994 	return objp;
2995 }
2996 #else
2997 #define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
2998 #endif
2999 
3000 static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
3001 {
3002 	if (cachep == kmem_cache)
3003 		return false;
3004 
3005 	return should_failslab(cachep->object_size, flags, cachep->flags);
3006 }
3007 
3008 static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3009 {
3010 	void *objp;
3011 	struct array_cache *ac;
3012 	bool force_refill = false;
3013 
3014 	check_irq_off();
3015 
3016 	ac = cpu_cache_get(cachep);
3017 	if (likely(ac->avail)) {
3018 		ac->touched = 1;
3019 		objp = ac_get_obj(cachep, ac, flags, false);
3020 
3021 		/*
3022 		 * Allow for the possibility all avail objects are not allowed
3023 		 * by the current flags
3024 		 */
3025 		if (objp) {
3026 			STATS_INC_ALLOCHIT(cachep);
3027 			goto out;
3028 		}
3029 		force_refill = true;
3030 	}
3031 
3032 	STATS_INC_ALLOCMISS(cachep);
3033 	objp = cache_alloc_refill(cachep, flags, force_refill);
3034 	/*
3035 	 * the 'ac' may be updated by cache_alloc_refill(),
3036 	 * and kmemleak_erase() requires its correct value.
3037 	 */
3038 	ac = cpu_cache_get(cachep);
3039 
3040 out:
3041 	/*
3042 	 * To avoid a false negative, if an object that is in one of the
3043 	 * per-CPU caches is leaked, we need to make sure kmemleak doesn't
3044 	 * treat the array pointers as a reference to the object.
3045 	 */
3046 	if (objp)
3047 		kmemleak_erase(&ac->entry[ac->avail]);
3048 	return objp;
3049 }
3050 
3051 #ifdef CONFIG_NUMA
3052 /*
3053  * Try allocating on another node if PF_SPREAD_SLAB is a mempolicy is set.
3054  *
3055  * If we are in_interrupt, then process context, including cpusets and
3056  * mempolicy, may not apply and should not be used for allocation policy.
3057  */
3058 static void *alternate_node_alloc(struct kmem_cache *cachep, gfp_t flags)
3059 {
3060 	int nid_alloc, nid_here;
3061 
3062 	if (in_interrupt() || (flags & __GFP_THISNODE))
3063 		return NULL;
3064 	nid_alloc = nid_here = numa_mem_id();
3065 	if (cpuset_do_slab_mem_spread() && (cachep->flags & SLAB_MEM_SPREAD))
3066 		nid_alloc = cpuset_slab_spread_node();
3067 	else if (current->mempolicy)
3068 		nid_alloc = mempolicy_slab_node();
3069 	if (nid_alloc != nid_here)
3070 		return ____cache_alloc_node(cachep, flags, nid_alloc);
3071 	return NULL;
3072 }
3073 
3074 /*
3075  * Fallback function if there was no memory available and no objects on a
3076  * certain node and fall back is permitted. First we scan all the
3077  * available node for available objects. If that fails then we
3078  * perform an allocation without specifying a node. This allows the page
3079  * allocator to do its reclaim / fallback magic. We then insert the
3080  * slab into the proper nodelist and then allocate from it.
3081  */
3082 static void *fallback_alloc(struct kmem_cache *cache, gfp_t flags)
3083 {
3084 	struct zonelist *zonelist;
3085 	gfp_t local_flags;
3086 	struct zoneref *z;
3087 	struct zone *zone;
3088 	enum zone_type high_zoneidx = gfp_zone(flags);
3089 	void *obj = NULL;
3090 	int nid;
3091 	unsigned int cpuset_mems_cookie;
3092 
3093 	if (flags & __GFP_THISNODE)
3094 		return NULL;
3095 
3096 	local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
3097 
3098 retry_cpuset:
3099 	cpuset_mems_cookie = read_mems_allowed_begin();
3100 	zonelist = node_zonelist(mempolicy_slab_node(), flags);
3101 
3102 retry:
3103 	/*
3104 	 * Look through allowed nodes for objects available
3105 	 * from existing per node queues.
3106 	 */
3107 	for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
3108 		nid = zone_to_nid(zone);
3109 
3110 		if (cpuset_zone_allowed_hardwall(zone, flags) &&
3111 			cache->node[nid] &&
3112 			cache->node[nid]->free_objects) {
3113 				obj = ____cache_alloc_node(cache,
3114 					flags | GFP_THISNODE, nid);
3115 				if (obj)
3116 					break;
3117 		}
3118 	}
3119 
3120 	if (!obj) {
3121 		/*
3122 		 * This allocation will be performed within the constraints
3123 		 * of the current cpuset / memory policy requirements.
3124 		 * We may trigger various forms of reclaim on the allowed
3125 		 * set and go into memory reserves if necessary.
3126 		 */
3127 		struct page *page;
3128 
3129 		if (local_flags & __GFP_WAIT)
3130 			local_irq_enable();
3131 		kmem_flagcheck(cache, flags);
3132 		page = kmem_getpages(cache, local_flags, numa_mem_id());
3133 		if (local_flags & __GFP_WAIT)
3134 			local_irq_disable();
3135 		if (page) {
3136 			/*
3137 			 * Insert into the appropriate per node queues
3138 			 */
3139 			nid = page_to_nid(page);
3140 			if (cache_grow(cache, flags, nid, page)) {
3141 				obj = ____cache_alloc_node(cache,
3142 					flags | GFP_THISNODE, nid);
3143 				if (!obj)
3144 					/*
3145 					 * Another processor may allocate the
3146 					 * objects in the slab since we are
3147 					 * not holding any locks.
3148 					 */
3149 					goto retry;
3150 			} else {
3151 				/* cache_grow already freed obj */
3152 				obj = NULL;
3153 			}
3154 		}
3155 	}
3156 
3157 	if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
3158 		goto retry_cpuset;
3159 	return obj;
3160 }
3161 
3162 /*
3163  * A interface to enable slab creation on nodeid
3164  */
3165 static void *____cache_alloc_node(struct kmem_cache *cachep, gfp_t flags,
3166 				int nodeid)
3167 {
3168 	struct list_head *entry;
3169 	struct page *page;
3170 	struct kmem_cache_node *n;
3171 	void *obj;
3172 	int x;
3173 
3174 	VM_BUG_ON(nodeid > num_online_nodes());
3175 	n = cachep->node[nodeid];
3176 	BUG_ON(!n);
3177 
3178 retry:
3179 	check_irq_off();
3180 	spin_lock(&n->list_lock);
3181 	entry = n->slabs_partial.next;
3182 	if (entry == &n->slabs_partial) {
3183 		n->free_touched = 1;
3184 		entry = n->slabs_free.next;
3185 		if (entry == &n->slabs_free)
3186 			goto must_grow;
3187 	}
3188 
3189 	page = list_entry(entry, struct page, lru);
3190 	check_spinlock_acquired_node(cachep, nodeid);
3191 
3192 	STATS_INC_NODEALLOCS(cachep);
3193 	STATS_INC_ACTIVE(cachep);
3194 	STATS_SET_HIGH(cachep);
3195 
3196 	BUG_ON(page->active == cachep->num);
3197 
3198 	obj = slab_get_obj(cachep, page, nodeid);
3199 	n->free_objects--;
3200 	/* move slabp to correct slabp list: */
3201 	list_del(&page->lru);
3202 
3203 	if (page->active == cachep->num)
3204 		list_add(&page->lru, &n->slabs_full);
3205 	else
3206 		list_add(&page->lru, &n->slabs_partial);
3207 
3208 	spin_unlock(&n->list_lock);
3209 	goto done;
3210 
3211 must_grow:
3212 	spin_unlock(&n->list_lock);
3213 	x = cache_grow(cachep, flags | GFP_THISNODE, nodeid, NULL);
3214 	if (x)
3215 		goto retry;
3216 
3217 	return fallback_alloc(cachep, flags);
3218 
3219 done:
3220 	return obj;
3221 }
3222 
3223 static __always_inline void *
3224 slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3225 		   unsigned long caller)
3226 {
3227 	unsigned long save_flags;
3228 	void *ptr;
3229 	int slab_node = numa_mem_id();
3230 
3231 	flags &= gfp_allowed_mask;
3232 
3233 	lockdep_trace_alloc(flags);
3234 
3235 	if (slab_should_failslab(cachep, flags))
3236 		return NULL;
3237 
3238 	cachep = memcg_kmem_get_cache(cachep, flags);
3239 
3240 	cache_alloc_debugcheck_before(cachep, flags);
3241 	local_irq_save(save_flags);
3242 
3243 	if (nodeid == NUMA_NO_NODE)
3244 		nodeid = slab_node;
3245 
3246 	if (unlikely(!cachep->node[nodeid])) {
3247 		/* Node not bootstrapped yet */
3248 		ptr = fallback_alloc(cachep, flags);
3249 		goto out;
3250 	}
3251 
3252 	if (nodeid == slab_node) {
3253 		/*
3254 		 * Use the locally cached objects if possible.
3255 		 * However ____cache_alloc does not allow fallback
3256 		 * to other nodes. It may fail while we still have
3257 		 * objects on other nodes available.
3258 		 */
3259 		ptr = ____cache_alloc(cachep, flags);
3260 		if (ptr)
3261 			goto out;
3262 	}
3263 	/* ___cache_alloc_node can fall back to other nodes */
3264 	ptr = ____cache_alloc_node(cachep, flags, nodeid);
3265   out:
3266 	local_irq_restore(save_flags);
3267 	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
3268 	kmemleak_alloc_recursive(ptr, cachep->object_size, 1, cachep->flags,
3269 				 flags);
3270 
3271 	if (likely(ptr)) {
3272 		kmemcheck_slab_alloc(cachep, flags, ptr, cachep->object_size);
3273 		if (unlikely(flags & __GFP_ZERO))
3274 			memset(ptr, 0, cachep->object_size);
3275 	}
3276 
3277 	return ptr;
3278 }
3279 
3280 static __always_inline void *
3281 __do_cache_alloc(struct kmem_cache *cache, gfp_t flags)
3282 {
3283 	void *objp;
3284 
3285 	if (current->mempolicy || unlikely(current->flags & PF_SPREAD_SLAB)) {
3286 		objp = alternate_node_alloc(cache, flags);
3287 		if (objp)
3288 			goto out;
3289 	}
3290 	objp = ____cache_alloc(cache, flags);
3291 
3292 	/*
3293 	 * We may just have run out of memory on the local node.
3294 	 * ____cache_alloc_node() knows how to locate memory on other nodes
3295 	 */
3296 	if (!objp)
3297 		objp = ____cache_alloc_node(cache, flags, numa_mem_id());
3298 
3299   out:
3300 	return objp;
3301 }
3302 #else
3303 
3304 static __always_inline void *
3305 __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3306 {
3307 	return ____cache_alloc(cachep, flags);
3308 }
3309 
3310 #endif /* CONFIG_NUMA */
3311 
3312 static __always_inline void *
3313 slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
3314 {
3315 	unsigned long save_flags;
3316 	void *objp;
3317 
3318 	flags &= gfp_allowed_mask;
3319 
3320 	lockdep_trace_alloc(flags);
3321 
3322 	if (slab_should_failslab(cachep, flags))
3323 		return NULL;
3324 
3325 	cachep = memcg_kmem_get_cache(cachep, flags);
3326 
3327 	cache_alloc_debugcheck_before(cachep, flags);
3328 	local_irq_save(save_flags);
3329 	objp = __do_cache_alloc(cachep, flags);
3330 	local_irq_restore(save_flags);
3331 	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3332 	kmemleak_alloc_recursive(objp, cachep->object_size, 1, cachep->flags,
3333 				 flags);
3334 	prefetchw(objp);
3335 
3336 	if (likely(objp)) {
3337 		kmemcheck_slab_alloc(cachep, flags, objp, cachep->object_size);
3338 		if (unlikely(flags & __GFP_ZERO))
3339 			memset(objp, 0, cachep->object_size);
3340 	}
3341 
3342 	return objp;
3343 }
3344 
3345 /*
3346  * Caller needs to acquire correct kmem_cache_node's list_lock
3347  */
3348 static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
3349 		       int node)
3350 {
3351 	int i;
3352 	struct kmem_cache_node *n;
3353 
3354 	for (i = 0; i < nr_objects; i++) {
3355 		void *objp;
3356 		struct page *page;
3357 
3358 		clear_obj_pfmemalloc(&objpp[i]);
3359 		objp = objpp[i];
3360 
3361 		page = virt_to_head_page(objp);
3362 		n = cachep->node[node];
3363 		list_del(&page->lru);
3364 		check_spinlock_acquired_node(cachep, node);
3365 		slab_put_obj(cachep, page, objp, node);
3366 		STATS_DEC_ACTIVE(cachep);
3367 		n->free_objects++;
3368 
3369 		/* fixup slab chains */
3370 		if (page->active == 0) {
3371 			if (n->free_objects > n->free_limit) {
3372 				n->free_objects -= cachep->num;
3373 				/* No need to drop any previously held
3374 				 * lock here, even if we have a off-slab slab
3375 				 * descriptor it is guaranteed to come from
3376 				 * a different cache, refer to comments before
3377 				 * alloc_slabmgmt.
3378 				 */
3379 				slab_destroy(cachep, page);
3380 			} else {
3381 				list_add(&page->lru, &n->slabs_free);
3382 			}
3383 		} else {
3384 			/* Unconditionally move a slab to the end of the
3385 			 * partial list on free - maximum time for the
3386 			 * other objects to be freed, too.
3387 			 */
3388 			list_add_tail(&page->lru, &n->slabs_partial);
3389 		}
3390 	}
3391 }
3392 
3393 static void cache_flusharray(struct kmem_cache *cachep, struct array_cache *ac)
3394 {
3395 	int batchcount;
3396 	struct kmem_cache_node *n;
3397 	int node = numa_mem_id();
3398 
3399 	batchcount = ac->batchcount;
3400 #if DEBUG
3401 	BUG_ON(!batchcount || batchcount > ac->avail);
3402 #endif
3403 	check_irq_off();
3404 	n = cachep->node[node];
3405 	spin_lock(&n->list_lock);
3406 	if (n->shared) {
3407 		struct array_cache *shared_array = n->shared;
3408 		int max = shared_array->limit - shared_array->avail;
3409 		if (max) {
3410 			if (batchcount > max)
3411 				batchcount = max;
3412 			memcpy(&(shared_array->entry[shared_array->avail]),
3413 			       ac->entry, sizeof(void *) * batchcount);
3414 			shared_array->avail += batchcount;
3415 			goto free_done;
3416 		}
3417 	}
3418 
3419 	free_block(cachep, ac->entry, batchcount, node);
3420 free_done:
3421 #if STATS
3422 	{
3423 		int i = 0;
3424 		struct list_head *p;
3425 
3426 		p = n->slabs_free.next;
3427 		while (p != &(n->slabs_free)) {
3428 			struct page *page;
3429 
3430 			page = list_entry(p, struct page, lru);
3431 			BUG_ON(page->active);
3432 
3433 			i++;
3434 			p = p->next;
3435 		}
3436 		STATS_SET_FREEABLE(cachep, i);
3437 	}
3438 #endif
3439 	spin_unlock(&n->list_lock);
3440 	ac->avail -= batchcount;
3441 	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void *)*ac->avail);
3442 }
3443 
3444 /*
3445  * Release an obj back to its cache. If the obj has a constructed state, it must
3446  * be in this state _before_ it is released.  Called with disabled ints.
3447  */
3448 static inline void __cache_free(struct kmem_cache *cachep, void *objp,
3449 				unsigned long caller)
3450 {
3451 	struct array_cache *ac = cpu_cache_get(cachep);
3452 
3453 	check_irq_off();
3454 	kmemleak_free_recursive(objp, cachep->flags);
3455 	objp = cache_free_debugcheck(cachep, objp, caller);
3456 
3457 	kmemcheck_slab_free(cachep, objp, cachep->object_size);
3458 
3459 	/*
3460 	 * Skip calling cache_free_alien() when the platform is not numa.
3461 	 * This will avoid cache misses that happen while accessing slabp (which
3462 	 * is per page memory  reference) to get nodeid. Instead use a global
3463 	 * variable to skip the call, which is mostly likely to be present in
3464 	 * the cache.
3465 	 */
3466 	if (nr_online_nodes > 1 && cache_free_alien(cachep, objp))
3467 		return;
3468 
3469 	if (likely(ac->avail < ac->limit)) {
3470 		STATS_INC_FREEHIT(cachep);
3471 	} else {
3472 		STATS_INC_FREEMISS(cachep);
3473 		cache_flusharray(cachep, ac);
3474 	}
3475 
3476 	ac_put_obj(cachep, ac, objp);
3477 }
3478 
3479 /**
3480  * kmem_cache_alloc - Allocate an object
3481  * @cachep: The cache to allocate from.
3482  * @flags: See kmalloc().
3483  *
3484  * Allocate an object from this cache.  The flags are only relevant
3485  * if the cache has no available objects.
3486  */
3487 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3488 {
3489 	void *ret = slab_alloc(cachep, flags, _RET_IP_);
3490 
3491 	trace_kmem_cache_alloc(_RET_IP_, ret,
3492 			       cachep->object_size, cachep->size, flags);
3493 
3494 	return ret;
3495 }
3496 EXPORT_SYMBOL(kmem_cache_alloc);
3497 
3498 #ifdef CONFIG_TRACING
3499 void *
3500 kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
3501 {
3502 	void *ret;
3503 
3504 	ret = slab_alloc(cachep, flags, _RET_IP_);
3505 
3506 	trace_kmalloc(_RET_IP_, ret,
3507 		      size, cachep->size, flags);
3508 	return ret;
3509 }
3510 EXPORT_SYMBOL(kmem_cache_alloc_trace);
3511 #endif
3512 
3513 #ifdef CONFIG_NUMA
3514 /**
3515  * kmem_cache_alloc_node - Allocate an object on the specified node
3516  * @cachep: The cache to allocate from.
3517  * @flags: See kmalloc().
3518  * @nodeid: node number of the target node.
3519  *
3520  * Identical to kmem_cache_alloc but it will allocate memory on the given
3521  * node, which can improve the performance for cpu bound structures.
3522  *
3523  * Fallback to other node is possible if __GFP_THISNODE is not set.
3524  */
3525 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
3526 {
3527 	void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3528 
3529 	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3530 				    cachep->object_size, cachep->size,
3531 				    flags, nodeid);
3532 
3533 	return ret;
3534 }
3535 EXPORT_SYMBOL(kmem_cache_alloc_node);
3536 
3537 #ifdef CONFIG_TRACING
3538 void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
3539 				  gfp_t flags,
3540 				  int nodeid,
3541 				  size_t size)
3542 {
3543 	void *ret;
3544 
3545 	ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
3546 
3547 	trace_kmalloc_node(_RET_IP_, ret,
3548 			   size, cachep->size,
3549 			   flags, nodeid);
3550 	return ret;
3551 }
3552 EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
3553 #endif
3554 
3555 static __always_inline void *
3556 __do_kmalloc_node(size_t size, gfp_t flags, int node, unsigned long caller)
3557 {
3558 	struct kmem_cache *cachep;
3559 
3560 	cachep = kmalloc_slab(size, flags);
3561 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3562 		return cachep;
3563 	return kmem_cache_alloc_node_trace(cachep, flags, node, size);
3564 }
3565 
3566 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
3567 void *__kmalloc_node(size_t size, gfp_t flags, int node)
3568 {
3569 	return __do_kmalloc_node(size, flags, node, _RET_IP_);
3570 }
3571 EXPORT_SYMBOL(__kmalloc_node);
3572 
3573 void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3574 		int node, unsigned long caller)
3575 {
3576 	return __do_kmalloc_node(size, flags, node, caller);
3577 }
3578 EXPORT_SYMBOL(__kmalloc_node_track_caller);
3579 #else
3580 void *__kmalloc_node(size_t size, gfp_t flags, int node)
3581 {
3582 	return __do_kmalloc_node(size, flags, node, 0);
3583 }
3584 EXPORT_SYMBOL(__kmalloc_node);
3585 #endif /* CONFIG_DEBUG_SLAB || CONFIG_TRACING */
3586 #endif /* CONFIG_NUMA */
3587 
3588 /**
3589  * __do_kmalloc - allocate memory
3590  * @size: how many bytes of memory are required.
3591  * @flags: the type of memory to allocate (see kmalloc).
3592  * @caller: function caller for debug tracking of the caller
3593  */
3594 static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3595 					  unsigned long caller)
3596 {
3597 	struct kmem_cache *cachep;
3598 	void *ret;
3599 
3600 	cachep = kmalloc_slab(size, flags);
3601 	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3602 		return cachep;
3603 	ret = slab_alloc(cachep, flags, caller);
3604 
3605 	trace_kmalloc(caller, ret,
3606 		      size, cachep->size, flags);
3607 
3608 	return ret;
3609 }
3610 
3611 
3612 #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_TRACING)
3613 void *__kmalloc(size_t size, gfp_t flags)
3614 {
3615 	return __do_kmalloc(size, flags, _RET_IP_);
3616 }
3617 EXPORT_SYMBOL(__kmalloc);
3618 
3619 void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3620 {
3621 	return __do_kmalloc(size, flags, caller);
3622 }
3623 EXPORT_SYMBOL(__kmalloc_track_caller);
3624 
3625 #else
3626 void *__kmalloc(size_t size, gfp_t flags)
3627 {
3628 	return __do_kmalloc(size, flags, 0);
3629 }
3630 EXPORT_SYMBOL(__kmalloc);
3631 #endif
3632 
3633 /**
3634  * kmem_cache_free - Deallocate an object
3635  * @cachep: The cache the allocation was from.
3636  * @objp: The previously allocated object.
3637  *
3638  * Free an object which was previously allocated from this
3639  * cache.
3640  */
3641 void kmem_cache_free(struct kmem_cache *cachep, void *objp)
3642 {
3643 	unsigned long flags;
3644 	cachep = cache_from_obj(cachep, objp);
3645 	if (!cachep)
3646 		return;
3647 
3648 	local_irq_save(flags);
3649 	debug_check_no_locks_freed(objp, cachep->object_size);
3650 	if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
3651 		debug_check_no_obj_freed(objp, cachep->object_size);
3652 	__cache_free(cachep, objp, _RET_IP_);
3653 	local_irq_restore(flags);
3654 
3655 	trace_kmem_cache_free(_RET_IP_, objp);
3656 }
3657 EXPORT_SYMBOL(kmem_cache_free);
3658 
3659 /**
3660  * kfree - free previously allocated memory
3661  * @objp: pointer returned by kmalloc.
3662  *
3663  * If @objp is NULL, no operation is performed.
3664  *
3665  * Don't free memory not originally allocated by kmalloc()
3666  * or you will run into trouble.
3667  */
3668 void kfree(const void *objp)
3669 {
3670 	struct kmem_cache *c;
3671 	unsigned long flags;
3672 
3673 	trace_kfree(_RET_IP_, objp);
3674 
3675 	if (unlikely(ZERO_OR_NULL_PTR(objp)))
3676 		return;
3677 	local_irq_save(flags);
3678 	kfree_debugcheck(objp);
3679 	c = virt_to_cache(objp);
3680 	debug_check_no_locks_freed(objp, c->object_size);
3681 
3682 	debug_check_no_obj_freed(objp, c->object_size);
3683 	__cache_free(c, (void *)objp, _RET_IP_);
3684 	local_irq_restore(flags);
3685 }
3686 EXPORT_SYMBOL(kfree);
3687 
3688 /*
3689  * This initializes kmem_cache_node or resizes various caches for all nodes.
3690  */
3691 static int alloc_kmem_cache_node(struct kmem_cache *cachep, gfp_t gfp)
3692 {
3693 	int node;
3694 	struct kmem_cache_node *n;
3695 	struct array_cache *new_shared;
3696 	struct array_cache **new_alien = NULL;
3697 
3698 	for_each_online_node(node) {
3699 
3700                 if (use_alien_caches) {
3701                         new_alien = alloc_alien_cache(node, cachep->limit, gfp);
3702                         if (!new_alien)
3703                                 goto fail;
3704                 }
3705 
3706 		new_shared = NULL;
3707 		if (cachep->shared) {
3708 			new_shared = alloc_arraycache(node,
3709 				cachep->shared*cachep->batchcount,
3710 					0xbaadf00d, gfp);
3711 			if (!new_shared) {
3712 				free_alien_cache(new_alien);
3713 				goto fail;
3714 			}
3715 		}
3716 
3717 		n = cachep->node[node];
3718 		if (n) {
3719 			struct array_cache *shared = n->shared;
3720 
3721 			spin_lock_irq(&n->list_lock);
3722 
3723 			if (shared)
3724 				free_block(cachep, shared->entry,
3725 						shared->avail, node);
3726 
3727 			n->shared = new_shared;
3728 			if (!n->alien) {
3729 				n->alien = new_alien;
3730 				new_alien = NULL;
3731 			}
3732 			n->free_limit = (1 + nr_cpus_node(node)) *
3733 					cachep->batchcount + cachep->num;
3734 			spin_unlock_irq(&n->list_lock);
3735 			kfree(shared);
3736 			free_alien_cache(new_alien);
3737 			continue;
3738 		}
3739 		n = kmalloc_node(sizeof(struct kmem_cache_node), gfp, node);
3740 		if (!n) {
3741 			free_alien_cache(new_alien);
3742 			kfree(new_shared);
3743 			goto fail;
3744 		}
3745 
3746 		kmem_cache_node_init(n);
3747 		n->next_reap = jiffies + REAPTIMEOUT_NODE +
3748 				((unsigned long)cachep) % REAPTIMEOUT_NODE;
3749 		n->shared = new_shared;
3750 		n->alien = new_alien;
3751 		n->free_limit = (1 + nr_cpus_node(node)) *
3752 					cachep->batchcount + cachep->num;
3753 		cachep->node[node] = n;
3754 	}
3755 	return 0;
3756 
3757 fail:
3758 	if (!cachep->list.next) {
3759 		/* Cache is not active yet. Roll back what we did */
3760 		node--;
3761 		while (node >= 0) {
3762 			if (cachep->node[node]) {
3763 				n = cachep->node[node];
3764 
3765 				kfree(n->shared);
3766 				free_alien_cache(n->alien);
3767 				kfree(n);
3768 				cachep->node[node] = NULL;
3769 			}
3770 			node--;
3771 		}
3772 	}
3773 	return -ENOMEM;
3774 }
3775 
3776 struct ccupdate_struct {
3777 	struct kmem_cache *cachep;
3778 	struct array_cache *new[0];
3779 };
3780 
3781 static void do_ccupdate_local(void *info)
3782 {
3783 	struct ccupdate_struct *new = info;
3784 	struct array_cache *old;
3785 
3786 	check_irq_off();
3787 	old = cpu_cache_get(new->cachep);
3788 
3789 	new->cachep->array[smp_processor_id()] = new->new[smp_processor_id()];
3790 	new->new[smp_processor_id()] = old;
3791 }
3792 
3793 /* Always called with the slab_mutex held */
3794 static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3795 				int batchcount, int shared, gfp_t gfp)
3796 {
3797 	struct ccupdate_struct *new;
3798 	int i;
3799 
3800 	new = kzalloc(sizeof(*new) + nr_cpu_ids * sizeof(struct array_cache *),
3801 		      gfp);
3802 	if (!new)
3803 		return -ENOMEM;
3804 
3805 	for_each_online_cpu(i) {
3806 		new->new[i] = alloc_arraycache(cpu_to_mem(i), limit,
3807 						batchcount, gfp);
3808 		if (!new->new[i]) {
3809 			for (i--; i >= 0; i--)
3810 				kfree(new->new[i]);
3811 			kfree(new);
3812 			return -ENOMEM;
3813 		}
3814 	}
3815 	new->cachep = cachep;
3816 
3817 	on_each_cpu(do_ccupdate_local, (void *)new, 1);
3818 
3819 	check_irq_on();
3820 	cachep->batchcount = batchcount;
3821 	cachep->limit = limit;
3822 	cachep->shared = shared;
3823 
3824 	for_each_online_cpu(i) {
3825 		struct array_cache *ccold = new->new[i];
3826 		if (!ccold)
3827 			continue;
3828 		spin_lock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
3829 		free_block(cachep, ccold->entry, ccold->avail, cpu_to_mem(i));
3830 		spin_unlock_irq(&cachep->node[cpu_to_mem(i)]->list_lock);
3831 		kfree(ccold);
3832 	}
3833 	kfree(new);
3834 	return alloc_kmem_cache_node(cachep, gfp);
3835 }
3836 
3837 static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3838 				int batchcount, int shared, gfp_t gfp)
3839 {
3840 	int ret;
3841 	struct kmem_cache *c = NULL;
3842 	int i = 0;
3843 
3844 	ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3845 
3846 	if (slab_state < FULL)
3847 		return ret;
3848 
3849 	if ((ret < 0) || !is_root_cache(cachep))
3850 		return ret;
3851 
3852 	VM_BUG_ON(!mutex_is_locked(&slab_mutex));
3853 	for_each_memcg_cache_index(i) {
3854 		c = cache_from_memcg_idx(cachep, i);
3855 		if (c)
3856 			/* return value determined by the parent cache only */
3857 			__do_tune_cpucache(c, limit, batchcount, shared, gfp);
3858 	}
3859 
3860 	return ret;
3861 }
3862 
3863 /* Called with slab_mutex held always */
3864 static int enable_cpucache(struct kmem_cache *cachep, gfp_t gfp)
3865 {
3866 	int err;
3867 	int limit = 0;
3868 	int shared = 0;
3869 	int batchcount = 0;
3870 
3871 	if (!is_root_cache(cachep)) {
3872 		struct kmem_cache *root = memcg_root_cache(cachep);
3873 		limit = root->limit;
3874 		shared = root->shared;
3875 		batchcount = root->batchcount;
3876 	}
3877 
3878 	if (limit && shared && batchcount)
3879 		goto skip_setup;
3880 	/*
3881 	 * The head array serves three purposes:
3882 	 * - create a LIFO ordering, i.e. return objects that are cache-warm
3883 	 * - reduce the number of spinlock operations.
3884 	 * - reduce the number of linked list operations on the slab and
3885 	 *   bufctl chains: array operations are cheaper.
3886 	 * The numbers are guessed, we should auto-tune as described by
3887 	 * Bonwick.
3888 	 */
3889 	if (cachep->size > 131072)
3890 		limit = 1;
3891 	else if (cachep->size > PAGE_SIZE)
3892 		limit = 8;
3893 	else if (cachep->size > 1024)
3894 		limit = 24;
3895 	else if (cachep->size > 256)
3896 		limit = 54;
3897 	else
3898 		limit = 120;
3899 
3900 	/*
3901 	 * CPU bound tasks (e.g. network routing) can exhibit cpu bound
3902 	 * allocation behaviour: Most allocs on one cpu, most free operations
3903 	 * on another cpu. For these cases, an efficient object passing between
3904 	 * cpus is necessary. This is provided by a shared array. The array
3905 	 * replaces Bonwick's magazine layer.
3906 	 * On uniprocessor, it's functionally equivalent (but less efficient)
3907 	 * to a larger limit. Thus disabled by default.
3908 	 */
3909 	shared = 0;
3910 	if (cachep->size <= PAGE_SIZE && num_possible_cpus() > 1)
3911 		shared = 8;
3912 
3913 #if DEBUG
3914 	/*
3915 	 * With debugging enabled, large batchcount lead to excessively long
3916 	 * periods with disabled local interrupts. Limit the batchcount
3917 	 */
3918 	if (limit > 32)
3919 		limit = 32;
3920 #endif
3921 	batchcount = (limit + 1) / 2;
3922 skip_setup:
3923 	err = do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3924 	if (err)
3925 		printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
3926 		       cachep->name, -err);
3927 	return err;
3928 }
3929 
3930 /*
3931  * Drain an array if it contains any elements taking the node lock only if
3932  * necessary. Note that the node listlock also protects the array_cache
3933  * if drain_array() is used on the shared array.
3934  */
3935 static void drain_array(struct kmem_cache *cachep, struct kmem_cache_node *n,
3936 			 struct array_cache *ac, int force, int node)
3937 {
3938 	int tofree;
3939 
3940 	if (!ac || !ac->avail)
3941 		return;
3942 	if (ac->touched && !force) {
3943 		ac->touched = 0;
3944 	} else {
3945 		spin_lock_irq(&n->list_lock);
3946 		if (ac->avail) {
3947 			tofree = force ? ac->avail : (ac->limit + 4) / 5;
3948 			if (tofree > ac->avail)
3949 				tofree = (ac->avail + 1) / 2;
3950 			free_block(cachep, ac->entry, tofree, node);
3951 			ac->avail -= tofree;
3952 			memmove(ac->entry, &(ac->entry[tofree]),
3953 				sizeof(void *) * ac->avail);
3954 		}
3955 		spin_unlock_irq(&n->list_lock);
3956 	}
3957 }
3958 
3959 /**
3960  * cache_reap - Reclaim memory from caches.
3961  * @w: work descriptor
3962  *
3963  * Called from workqueue/eventd every few seconds.
3964  * Purpose:
3965  * - clear the per-cpu caches for this CPU.
3966  * - return freeable pages to the main free memory pool.
3967  *
3968  * If we cannot acquire the cache chain mutex then just give up - we'll try
3969  * again on the next iteration.
3970  */
3971 static void cache_reap(struct work_struct *w)
3972 {
3973 	struct kmem_cache *searchp;
3974 	struct kmem_cache_node *n;
3975 	int node = numa_mem_id();
3976 	struct delayed_work *work = to_delayed_work(w);
3977 
3978 	if (!mutex_trylock(&slab_mutex))
3979 		/* Give up. Setup the next iteration. */
3980 		goto out;
3981 
3982 	list_for_each_entry(searchp, &slab_caches, list) {
3983 		check_irq_on();
3984 
3985 		/*
3986 		 * We only take the node lock if absolutely necessary and we
3987 		 * have established with reasonable certainty that
3988 		 * we can do some work if the lock was obtained.
3989 		 */
3990 		n = searchp->node[node];
3991 
3992 		reap_alien(searchp, n);
3993 
3994 		drain_array(searchp, n, cpu_cache_get(searchp), 0, node);
3995 
3996 		/*
3997 		 * These are racy checks but it does not matter
3998 		 * if we skip one check or scan twice.
3999 		 */
4000 		if (time_after(n->next_reap, jiffies))
4001 			goto next;
4002 
4003 		n->next_reap = jiffies + REAPTIMEOUT_NODE;
4004 
4005 		drain_array(searchp, n, n->shared, 0, node);
4006 
4007 		if (n->free_touched)
4008 			n->free_touched = 0;
4009 		else {
4010 			int freed;
4011 
4012 			freed = drain_freelist(searchp, n, (n->free_limit +
4013 				5 * searchp->num - 1) / (5 * searchp->num));
4014 			STATS_ADD_REAPED(searchp, freed);
4015 		}
4016 next:
4017 		cond_resched();
4018 	}
4019 	check_irq_on();
4020 	mutex_unlock(&slab_mutex);
4021 	next_reap_node();
4022 out:
4023 	/* Set up the next iteration */
4024 	schedule_delayed_work(work, round_jiffies_relative(REAPTIMEOUT_AC));
4025 }
4026 
4027 #ifdef CONFIG_SLABINFO
4028 void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
4029 {
4030 	struct page *page;
4031 	unsigned long active_objs;
4032 	unsigned long num_objs;
4033 	unsigned long active_slabs = 0;
4034 	unsigned long num_slabs, free_objects = 0, shared_avail = 0;
4035 	const char *name;
4036 	char *error = NULL;
4037 	int node;
4038 	struct kmem_cache_node *n;
4039 
4040 	active_objs = 0;
4041 	num_slabs = 0;
4042 	for_each_online_node(node) {
4043 		n = cachep->node[node];
4044 		if (!n)
4045 			continue;
4046 
4047 		check_irq_on();
4048 		spin_lock_irq(&n->list_lock);
4049 
4050 		list_for_each_entry(page, &n->slabs_full, lru) {
4051 			if (page->active != cachep->num && !error)
4052 				error = "slabs_full accounting error";
4053 			active_objs += cachep->num;
4054 			active_slabs++;
4055 		}
4056 		list_for_each_entry(page, &n->slabs_partial, lru) {
4057 			if (page->active == cachep->num && !error)
4058 				error = "slabs_partial accounting error";
4059 			if (!page->active && !error)
4060 				error = "slabs_partial accounting error";
4061 			active_objs += page->active;
4062 			active_slabs++;
4063 		}
4064 		list_for_each_entry(page, &n->slabs_free, lru) {
4065 			if (page->active && !error)
4066 				error = "slabs_free accounting error";
4067 			num_slabs++;
4068 		}
4069 		free_objects += n->free_objects;
4070 		if (n->shared)
4071 			shared_avail += n->shared->avail;
4072 
4073 		spin_unlock_irq(&n->list_lock);
4074 	}
4075 	num_slabs += active_slabs;
4076 	num_objs = num_slabs * cachep->num;
4077 	if (num_objs - active_objs != free_objects && !error)
4078 		error = "free_objects accounting error";
4079 
4080 	name = cachep->name;
4081 	if (error)
4082 		printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4083 
4084 	sinfo->active_objs = active_objs;
4085 	sinfo->num_objs = num_objs;
4086 	sinfo->active_slabs = active_slabs;
4087 	sinfo->num_slabs = num_slabs;
4088 	sinfo->shared_avail = shared_avail;
4089 	sinfo->limit = cachep->limit;
4090 	sinfo->batchcount = cachep->batchcount;
4091 	sinfo->shared = cachep->shared;
4092 	sinfo->objects_per_slab = cachep->num;
4093 	sinfo->cache_order = cachep->gfporder;
4094 }
4095 
4096 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4097 {
4098 #if STATS
4099 	{			/* node stats */
4100 		unsigned long high = cachep->high_mark;
4101 		unsigned long allocs = cachep->num_allocations;
4102 		unsigned long grown = cachep->grown;
4103 		unsigned long reaped = cachep->reaped;
4104 		unsigned long errors = cachep->errors;
4105 		unsigned long max_freeable = cachep->max_freeable;
4106 		unsigned long node_allocs = cachep->node_allocs;
4107 		unsigned long node_frees = cachep->node_frees;
4108 		unsigned long overflows = cachep->node_overflow;
4109 
4110 		seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu "
4111 			   "%4lu %4lu %4lu %4lu %4lu",
4112 			   allocs, high, grown,
4113 			   reaped, errors, max_freeable, node_allocs,
4114 			   node_frees, overflows);
4115 	}
4116 	/* cpu stats */
4117 	{
4118 		unsigned long allochit = atomic_read(&cachep->allochit);
4119 		unsigned long allocmiss = atomic_read(&cachep->allocmiss);
4120 		unsigned long freehit = atomic_read(&cachep->freehit);
4121 		unsigned long freemiss = atomic_read(&cachep->freemiss);
4122 
4123 		seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
4124 			   allochit, allocmiss, freehit, freemiss);
4125 	}
4126 #endif
4127 }
4128 
4129 #define MAX_SLABINFO_WRITE 128
4130 /**
4131  * slabinfo_write - Tuning for the slab allocator
4132  * @file: unused
4133  * @buffer: user buffer
4134  * @count: data length
4135  * @ppos: unused
4136  */
4137 ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4138 		       size_t count, loff_t *ppos)
4139 {
4140 	char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
4141 	int limit, batchcount, shared, res;
4142 	struct kmem_cache *cachep;
4143 
4144 	if (count > MAX_SLABINFO_WRITE)
4145 		return -EINVAL;
4146 	if (copy_from_user(&kbuf, buffer, count))
4147 		return -EFAULT;
4148 	kbuf[MAX_SLABINFO_WRITE] = '\0';
4149 
4150 	tmp = strchr(kbuf, ' ');
4151 	if (!tmp)
4152 		return -EINVAL;
4153 	*tmp = '\0';
4154 	tmp++;
4155 	if (sscanf(tmp, " %d %d %d", &limit, &batchcount, &shared) != 3)
4156 		return -EINVAL;
4157 
4158 	/* Find the cache in the chain of caches. */
4159 	mutex_lock(&slab_mutex);
4160 	res = -EINVAL;
4161 	list_for_each_entry(cachep, &slab_caches, list) {
4162 		if (!strcmp(cachep->name, kbuf)) {
4163 			if (limit < 1 || batchcount < 1 ||
4164 					batchcount > limit || shared < 0) {
4165 				res = 0;
4166 			} else {
4167 				res = do_tune_cpucache(cachep, limit,
4168 						       batchcount, shared,
4169 						       GFP_KERNEL);
4170 			}
4171 			break;
4172 		}
4173 	}
4174 	mutex_unlock(&slab_mutex);
4175 	if (res >= 0)
4176 		res = count;
4177 	return res;
4178 }
4179 
4180 #ifdef CONFIG_DEBUG_SLAB_LEAK
4181 
4182 static void *leaks_start(struct seq_file *m, loff_t *pos)
4183 {
4184 	mutex_lock(&slab_mutex);
4185 	return seq_list_start(&slab_caches, *pos);
4186 }
4187 
4188 static inline int add_caller(unsigned long *n, unsigned long v)
4189 {
4190 	unsigned long *p;
4191 	int l;
4192 	if (!v)
4193 		return 1;
4194 	l = n[1];
4195 	p = n + 2;
4196 	while (l) {
4197 		int i = l/2;
4198 		unsigned long *q = p + 2 * i;
4199 		if (*q == v) {
4200 			q[1]++;
4201 			return 1;
4202 		}
4203 		if (*q > v) {
4204 			l = i;
4205 		} else {
4206 			p = q + 2;
4207 			l -= i + 1;
4208 		}
4209 	}
4210 	if (++n[1] == n[0])
4211 		return 0;
4212 	memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void *)p - (void *)n));
4213 	p[0] = v;
4214 	p[1] = 1;
4215 	return 1;
4216 }
4217 
4218 static void handle_slab(unsigned long *n, struct kmem_cache *c,
4219 						struct page *page)
4220 {
4221 	void *p;
4222 	int i, j;
4223 
4224 	if (n[0] == n[1])
4225 		return;
4226 	for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4227 		bool active = true;
4228 
4229 		for (j = page->active; j < c->num; j++) {
4230 			/* Skip freed item */
4231 			if (get_free_obj(page, j) == i) {
4232 				active = false;
4233 				break;
4234 			}
4235 		}
4236 		if (!active)
4237 			continue;
4238 
4239 		if (!add_caller(n, (unsigned long)*dbg_userword(c, p)))
4240 			return;
4241 	}
4242 }
4243 
4244 static void show_symbol(struct seq_file *m, unsigned long address)
4245 {
4246 #ifdef CONFIG_KALLSYMS
4247 	unsigned long offset, size;
4248 	char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4249 
4250 	if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4251 		seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4252 		if (modname[0])
4253 			seq_printf(m, " [%s]", modname);
4254 		return;
4255 	}
4256 #endif
4257 	seq_printf(m, "%p", (void *)address);
4258 }
4259 
4260 static int leaks_show(struct seq_file *m, void *p)
4261 {
4262 	struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4263 	struct page *page;
4264 	struct kmem_cache_node *n;
4265 	const char *name;
4266 	unsigned long *x = m->private;
4267 	int node;
4268 	int i;
4269 
4270 	if (!(cachep->flags & SLAB_STORE_USER))
4271 		return 0;
4272 	if (!(cachep->flags & SLAB_RED_ZONE))
4273 		return 0;
4274 
4275 	/* OK, we can do it */
4276 
4277 	x[1] = 0;
4278 
4279 	for_each_online_node(node) {
4280 		n = cachep->node[node];
4281 		if (!n)
4282 			continue;
4283 
4284 		check_irq_on();
4285 		spin_lock_irq(&n->list_lock);
4286 
4287 		list_for_each_entry(page, &n->slabs_full, lru)
4288 			handle_slab(x, cachep, page);
4289 		list_for_each_entry(page, &n->slabs_partial, lru)
4290 			handle_slab(x, cachep, page);
4291 		spin_unlock_irq(&n->list_lock);
4292 	}
4293 	name = cachep->name;
4294 	if (x[0] == x[1]) {
4295 		/* Increase the buffer size */
4296 		mutex_unlock(&slab_mutex);
4297 		m->private = kzalloc(x[0] * 4 * sizeof(unsigned long), GFP_KERNEL);
4298 		if (!m->private) {
4299 			/* Too bad, we are really out */
4300 			m->private = x;
4301 			mutex_lock(&slab_mutex);
4302 			return -ENOMEM;
4303 		}
4304 		*(unsigned long *)m->private = x[0] * 2;
4305 		kfree(x);
4306 		mutex_lock(&slab_mutex);
4307 		/* Now make sure this entry will be retried */
4308 		m->count = m->size;
4309 		return 0;
4310 	}
4311 	for (i = 0; i < x[1]; i++) {
4312 		seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4313 		show_symbol(m, x[2*i+2]);
4314 		seq_putc(m, '\n');
4315 	}
4316 
4317 	return 0;
4318 }
4319 
4320 static const struct seq_operations slabstats_op = {
4321 	.start = leaks_start,
4322 	.next = slab_next,
4323 	.stop = slab_stop,
4324 	.show = leaks_show,
4325 };
4326 
4327 static int slabstats_open(struct inode *inode, struct file *file)
4328 {
4329 	unsigned long *n = kzalloc(PAGE_SIZE, GFP_KERNEL);
4330 	int ret = -ENOMEM;
4331 	if (n) {
4332 		ret = seq_open(file, &slabstats_op);
4333 		if (!ret) {
4334 			struct seq_file *m = file->private_data;
4335 			*n = PAGE_SIZE / (2 * sizeof(unsigned long));
4336 			m->private = n;
4337 			n = NULL;
4338 		}
4339 		kfree(n);
4340 	}
4341 	return ret;
4342 }
4343 
4344 static const struct file_operations proc_slabstats_operations = {
4345 	.open		= slabstats_open,
4346 	.read		= seq_read,
4347 	.llseek		= seq_lseek,
4348 	.release	= seq_release_private,
4349 };
4350 #endif
4351 
4352 static int __init slab_proc_init(void)
4353 {
4354 #ifdef CONFIG_DEBUG_SLAB_LEAK
4355 	proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4356 #endif
4357 	return 0;
4358 }
4359 module_init(slab_proc_init);
4360 #endif
4361 
4362 /**
4363  * ksize - get the actual amount of memory allocated for a given object
4364  * @objp: Pointer to the object
4365  *
4366  * kmalloc may internally round up allocations and return more memory
4367  * than requested. ksize() can be used to determine the actual amount of
4368  * memory allocated. The caller may use this additional memory, even though
4369  * a smaller amount of memory was initially specified with the kmalloc call.
4370  * The caller must guarantee that objp points to a valid object previously
4371  * allocated with either kmalloc() or kmem_cache_alloc(). The object
4372  * must not be freed during the duration of the call.
4373  */
4374 size_t ksize(const void *objp)
4375 {
4376 	BUG_ON(!objp);
4377 	if (unlikely(objp == ZERO_SIZE_PTR))
4378 		return 0;
4379 
4380 	return virt_to_cache(objp)->object_size;
4381 }
4382 EXPORT_SYMBOL(ksize);
4383