1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). 4 * 5 * (C) SGI 2006, Christoph Lameter 6 * Cleaned up and restructured to ease the addition of alternative 7 * implementations of SLAB allocators. 8 * (C) Linux Foundation 2008-2013 9 * Unified interface for all slab allocators 10 */ 11 12 #ifndef _LINUX_SLAB_H 13 #define _LINUX_SLAB_H 14 15 #include <linux/gfp.h> 16 #include <linux/overflow.h> 17 #include <linux/types.h> 18 #include <linux/workqueue.h> 19 20 21 /* 22 * Flags to pass to kmem_cache_create(). 23 * The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set. 24 */ 25 /* DEBUG: Perform (expensive) checks on alloc/free */ 26 #define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) 27 /* DEBUG: Red zone objs in a cache */ 28 #define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) 29 /* DEBUG: Poison objects */ 30 #define SLAB_POISON ((slab_flags_t __force)0x00000800U) 31 /* Align objs on cache lines */ 32 #define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) 33 /* Use GFP_DMA memory */ 34 #define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) 35 /* Use GFP_DMA32 memory */ 36 #define SLAB_CACHE_DMA32 ((slab_flags_t __force)0x00008000U) 37 /* DEBUG: Store the last owner for bug hunting */ 38 #define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) 39 /* Panic if kmem_cache_create() fails */ 40 #define SLAB_PANIC ((slab_flags_t __force)0x00040000U) 41 /* 42 * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 43 * 44 * This delays freeing the SLAB page by a grace period, it does _NOT_ 45 * delay object freeing. This means that if you do kmem_cache_free() 46 * that memory location is free to be reused at any time. Thus it may 47 * be possible to see another object there in the same RCU grace period. 48 * 49 * This feature only ensures the memory location backing the object 50 * stays valid, the trick to using this is relying on an independent 51 * object validation pass. Something like: 52 * 53 * rcu_read_lock() 54 * again: 55 * obj = lockless_lookup(key); 56 * if (obj) { 57 * if (!try_get_ref(obj)) // might fail for free objects 58 * goto again; 59 * 60 * if (obj->key != key) { // not the object we expected 61 * put_ref(obj); 62 * goto again; 63 * } 64 * } 65 * rcu_read_unlock(); 66 * 67 * This is useful if we need to approach a kernel structure obliquely, 68 * from its address obtained without the usual locking. We can lock 69 * the structure to stabilize it and check it's still at the given address, 70 * only if we can be sure that the memory has not been meanwhile reused 71 * for some other kind of object (which our subsystem's lock might corrupt). 72 * 73 * rcu_read_lock before reading the address, then rcu_read_unlock after 74 * taking the spinlock within the structure expected at that address. 75 * 76 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 77 */ 78 /* Defer freeing slabs to RCU */ 79 #define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) 80 /* Spread some memory over cpuset */ 81 #define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) 82 /* Trace allocations and frees */ 83 #define SLAB_TRACE ((slab_flags_t __force)0x00200000U) 84 85 /* Flag to prevent checks on free */ 86 #ifdef CONFIG_DEBUG_OBJECTS 87 # define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) 88 #else 89 # define SLAB_DEBUG_OBJECTS 0 90 #endif 91 92 /* Avoid kmemleak tracing */ 93 #define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) 94 95 /* Fault injection mark */ 96 #ifdef CONFIG_FAILSLAB 97 # define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) 98 #else 99 # define SLAB_FAILSLAB 0 100 #endif 101 /* Account to memcg */ 102 #ifdef CONFIG_MEMCG_KMEM 103 # define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) 104 #else 105 # define SLAB_ACCOUNT 0 106 #endif 107 108 #ifdef CONFIG_KASAN 109 #define SLAB_KASAN ((slab_flags_t __force)0x08000000U) 110 #else 111 #define SLAB_KASAN 0 112 #endif 113 114 /* The following flags affect the page allocator grouping pages by mobility */ 115 /* Objects are reclaimable */ 116 #define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) 117 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ 118 /* 119 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. 120 * 121 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. 122 * 123 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. 124 * Both make kfree a no-op. 125 */ 126 #define ZERO_SIZE_PTR ((void *)16) 127 128 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 129 (unsigned long)ZERO_SIZE_PTR) 130 131 #include <linux/kasan.h> 132 133 struct mem_cgroup; 134 /* 135 * struct kmem_cache related prototypes 136 */ 137 void __init kmem_cache_init(void); 138 bool slab_is_available(void); 139 140 extern bool usercopy_fallback; 141 142 struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, 143 unsigned int align, slab_flags_t flags, 144 void (*ctor)(void *)); 145 struct kmem_cache *kmem_cache_create_usercopy(const char *name, 146 unsigned int size, unsigned int align, 147 slab_flags_t flags, 148 unsigned int useroffset, unsigned int usersize, 149 void (*ctor)(void *)); 150 void kmem_cache_destroy(struct kmem_cache *); 151 int kmem_cache_shrink(struct kmem_cache *); 152 153 void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *); 154 void memcg_deactivate_kmem_caches(struct mem_cgroup *); 155 void memcg_destroy_kmem_caches(struct mem_cgroup *); 156 157 /* 158 * Please use this macro to create slab caches. Simply specify the 159 * name of the structure and maybe some flags that are listed above. 160 * 161 * The alignment of the struct determines object alignment. If you 162 * f.e. add ____cacheline_aligned_in_smp to the struct declaration 163 * then the objects will be properly aligned in SMP configurations. 164 */ 165 #define KMEM_CACHE(__struct, __flags) \ 166 kmem_cache_create(#__struct, sizeof(struct __struct), \ 167 __alignof__(struct __struct), (__flags), NULL) 168 169 /* 170 * To whitelist a single field for copying to/from usercopy, use this 171 * macro instead for KMEM_CACHE() above. 172 */ 173 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ 174 kmem_cache_create_usercopy(#__struct, \ 175 sizeof(struct __struct), \ 176 __alignof__(struct __struct), (__flags), \ 177 offsetof(struct __struct, __field), \ 178 sizeof_field(struct __struct, __field), NULL) 179 180 /* 181 * Common kmalloc functions provided by all allocators 182 */ 183 void * __must_check __krealloc(const void *, size_t, gfp_t); 184 void * __must_check krealloc(const void *, size_t, gfp_t); 185 void kfree(const void *); 186 void kzfree(const void *); 187 size_t ksize(const void *); 188 189 #ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR 190 void __check_heap_object(const void *ptr, unsigned long n, struct page *page, 191 bool to_user); 192 #else 193 static inline void __check_heap_object(const void *ptr, unsigned long n, 194 struct page *page, bool to_user) { } 195 #endif 196 197 /* 198 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 199 * alignment larger than the alignment of a 64-bit integer. 200 * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. 201 */ 202 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 203 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN 204 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN 205 #define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) 206 #else 207 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 208 #endif 209 210 /* 211 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. 212 * Intended for arches that get misalignment faults even for 64 bit integer 213 * aligned buffers. 214 */ 215 #ifndef ARCH_SLAB_MINALIGN 216 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 217 #endif 218 219 /* 220 * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned 221 * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN 222 * aligned pointers. 223 */ 224 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) 225 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) 226 #define __assume_page_alignment __assume_aligned(PAGE_SIZE) 227 228 /* 229 * Kmalloc array related definitions 230 */ 231 232 #ifdef CONFIG_SLAB 233 /* 234 * The largest kmalloc size supported by the SLAB allocators is 235 * 32 megabyte (2^25) or the maximum allocatable page order if that is 236 * less than 32 MB. 237 * 238 * WARNING: Its not easy to increase this value since the allocators have 239 * to do various tricks to work around compiler limitations in order to 240 * ensure proper constant folding. 241 */ 242 #define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ 243 (MAX_ORDER + PAGE_SHIFT - 1) : 25) 244 #define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH 245 #ifndef KMALLOC_SHIFT_LOW 246 #define KMALLOC_SHIFT_LOW 5 247 #endif 248 #endif 249 250 #ifdef CONFIG_SLUB 251 /* 252 * SLUB directly allocates requests fitting in to an order-1 page 253 * (PAGE_SIZE*2). Larger requests are passed to the page allocator. 254 */ 255 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 256 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 257 #ifndef KMALLOC_SHIFT_LOW 258 #define KMALLOC_SHIFT_LOW 3 259 #endif 260 #endif 261 262 #ifdef CONFIG_SLOB 263 /* 264 * SLOB passes all requests larger than one page to the page allocator. 265 * No kmalloc array is necessary since objects of different sizes can 266 * be allocated from the same page. 267 */ 268 #define KMALLOC_SHIFT_HIGH PAGE_SHIFT 269 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 270 #ifndef KMALLOC_SHIFT_LOW 271 #define KMALLOC_SHIFT_LOW 3 272 #endif 273 #endif 274 275 /* Maximum allocatable size */ 276 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) 277 /* Maximum size for which we actually use a slab cache */ 278 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) 279 /* Maximum order allocatable via the slab allocagtor */ 280 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) 281 282 /* 283 * Kmalloc subsystem. 284 */ 285 #ifndef KMALLOC_MIN_SIZE 286 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 287 #endif 288 289 /* 290 * This restriction comes from byte sized index implementation. 291 * Page size is normally 2^12 bytes and, in this case, if we want to use 292 * byte sized index which can represent 2^8 entries, the size of the object 293 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. 294 * If minimum size of kmalloc is less than 16, we use it as minimum object 295 * size and give up to use byte sized index. 296 */ 297 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ 298 (KMALLOC_MIN_SIZE) : 16) 299 300 /* 301 * Whenever changing this, take care of that kmalloc_type() and 302 * create_kmalloc_caches() still work as intended. 303 */ 304 enum kmalloc_cache_type { 305 KMALLOC_NORMAL = 0, 306 KMALLOC_RECLAIM, 307 #ifdef CONFIG_ZONE_DMA 308 KMALLOC_DMA, 309 #endif 310 NR_KMALLOC_TYPES 311 }; 312 313 #ifndef CONFIG_SLOB 314 extern struct kmem_cache * 315 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1]; 316 317 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags) 318 { 319 #ifdef CONFIG_ZONE_DMA 320 /* 321 * The most common case is KMALLOC_NORMAL, so test for it 322 * with a single branch for both flags. 323 */ 324 if (likely((flags & (__GFP_DMA | __GFP_RECLAIMABLE)) == 0)) 325 return KMALLOC_NORMAL; 326 327 /* 328 * At least one of the flags has to be set. If both are, __GFP_DMA 329 * is more important. 330 */ 331 return flags & __GFP_DMA ? KMALLOC_DMA : KMALLOC_RECLAIM; 332 #else 333 return flags & __GFP_RECLAIMABLE ? KMALLOC_RECLAIM : KMALLOC_NORMAL; 334 #endif 335 } 336 337 /* 338 * Figure out which kmalloc slab an allocation of a certain size 339 * belongs to. 340 * 0 = zero alloc 341 * 1 = 65 .. 96 bytes 342 * 2 = 129 .. 192 bytes 343 * n = 2^(n-1)+1 .. 2^n 344 */ 345 static __always_inline unsigned int kmalloc_index(size_t size) 346 { 347 if (!size) 348 return 0; 349 350 if (size <= KMALLOC_MIN_SIZE) 351 return KMALLOC_SHIFT_LOW; 352 353 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) 354 return 1; 355 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) 356 return 2; 357 if (size <= 8) return 3; 358 if (size <= 16) return 4; 359 if (size <= 32) return 5; 360 if (size <= 64) return 6; 361 if (size <= 128) return 7; 362 if (size <= 256) return 8; 363 if (size <= 512) return 9; 364 if (size <= 1024) return 10; 365 if (size <= 2 * 1024) return 11; 366 if (size <= 4 * 1024) return 12; 367 if (size <= 8 * 1024) return 13; 368 if (size <= 16 * 1024) return 14; 369 if (size <= 32 * 1024) return 15; 370 if (size <= 64 * 1024) return 16; 371 if (size <= 128 * 1024) return 17; 372 if (size <= 256 * 1024) return 18; 373 if (size <= 512 * 1024) return 19; 374 if (size <= 1024 * 1024) return 20; 375 if (size <= 2 * 1024 * 1024) return 21; 376 if (size <= 4 * 1024 * 1024) return 22; 377 if (size <= 8 * 1024 * 1024) return 23; 378 if (size <= 16 * 1024 * 1024) return 24; 379 if (size <= 32 * 1024 * 1024) return 25; 380 if (size <= 64 * 1024 * 1024) return 26; 381 BUG(); 382 383 /* Will never be reached. Needed because the compiler may complain */ 384 return -1; 385 } 386 #endif /* !CONFIG_SLOB */ 387 388 void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc; 389 void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags) __assume_slab_alignment __malloc; 390 void kmem_cache_free(struct kmem_cache *, void *); 391 392 /* 393 * Bulk allocation and freeing operations. These are accelerated in an 394 * allocator specific way to avoid taking locks repeatedly or building 395 * metadata structures unnecessarily. 396 * 397 * Note that interrupts must be enabled when calling these functions. 398 */ 399 void kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); 400 int kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); 401 402 /* 403 * Caller must not use kfree_bulk() on memory not originally allocated 404 * by kmalloc(), because the SLOB allocator cannot handle this. 405 */ 406 static __always_inline void kfree_bulk(size_t size, void **p) 407 { 408 kmem_cache_free_bulk(NULL, size, p); 409 } 410 411 #ifdef CONFIG_NUMA 412 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc; 413 void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc; 414 #else 415 static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) 416 { 417 return __kmalloc(size, flags); 418 } 419 420 static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) 421 { 422 return kmem_cache_alloc(s, flags); 423 } 424 #endif 425 426 #ifdef CONFIG_TRACING 427 extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t) __assume_slab_alignment __malloc; 428 429 #ifdef CONFIG_NUMA 430 extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 431 gfp_t gfpflags, 432 int node, size_t size) __assume_slab_alignment __malloc; 433 #else 434 static __always_inline void * 435 kmem_cache_alloc_node_trace(struct kmem_cache *s, 436 gfp_t gfpflags, 437 int node, size_t size) 438 { 439 return kmem_cache_alloc_trace(s, gfpflags, size); 440 } 441 #endif /* CONFIG_NUMA */ 442 443 #else /* CONFIG_TRACING */ 444 static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, 445 gfp_t flags, size_t size) 446 { 447 void *ret = kmem_cache_alloc(s, flags); 448 449 ret = kasan_kmalloc(s, ret, size, flags); 450 return ret; 451 } 452 453 static __always_inline void * 454 kmem_cache_alloc_node_trace(struct kmem_cache *s, 455 gfp_t gfpflags, 456 int node, size_t size) 457 { 458 void *ret = kmem_cache_alloc_node(s, gfpflags, node); 459 460 ret = kasan_kmalloc(s, ret, size, gfpflags); 461 return ret; 462 } 463 #endif /* CONFIG_TRACING */ 464 465 extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; 466 467 #ifdef CONFIG_TRACING 468 extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; 469 #else 470 static __always_inline void * 471 kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 472 { 473 return kmalloc_order(size, flags, order); 474 } 475 #endif 476 477 static __always_inline void *kmalloc_large(size_t size, gfp_t flags) 478 { 479 unsigned int order = get_order(size); 480 return kmalloc_order_trace(size, flags, order); 481 } 482 483 /** 484 * kmalloc - allocate memory 485 * @size: how many bytes of memory are required. 486 * @flags: the type of memory to allocate. 487 * 488 * kmalloc is the normal method of allocating memory 489 * for objects smaller than page size in the kernel. 490 * 491 * The @flags argument may be one of the GFP flags defined at 492 * include/linux/gfp.h and described at 493 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>` 494 * 495 * The recommended usage of the @flags is described at 496 * :ref:`Documentation/core-api/memory-allocation.rst <memory-allocation>` 497 * 498 * Below is a brief outline of the most useful GFP flags 499 * 500 * %GFP_KERNEL 501 * Allocate normal kernel ram. May sleep. 502 * 503 * %GFP_NOWAIT 504 * Allocation will not sleep. 505 * 506 * %GFP_ATOMIC 507 * Allocation will not sleep. May use emergency pools. 508 * 509 * %GFP_HIGHUSER 510 * Allocate memory from high memory on behalf of user. 511 * 512 * Also it is possible to set different flags by OR'ing 513 * in one or more of the following additional @flags: 514 * 515 * %__GFP_HIGH 516 * This allocation has high priority and may use emergency pools. 517 * 518 * %__GFP_NOFAIL 519 * Indicate that this allocation is in no way allowed to fail 520 * (think twice before using). 521 * 522 * %__GFP_NORETRY 523 * If memory is not immediately available, 524 * then give up at once. 525 * 526 * %__GFP_NOWARN 527 * If allocation fails, don't issue any warnings. 528 * 529 * %__GFP_RETRY_MAYFAIL 530 * Try really hard to succeed the allocation but fail 531 * eventually. 532 */ 533 static __always_inline void *kmalloc(size_t size, gfp_t flags) 534 { 535 if (__builtin_constant_p(size)) { 536 #ifndef CONFIG_SLOB 537 unsigned int index; 538 #endif 539 if (size > KMALLOC_MAX_CACHE_SIZE) 540 return kmalloc_large(size, flags); 541 #ifndef CONFIG_SLOB 542 index = kmalloc_index(size); 543 544 if (!index) 545 return ZERO_SIZE_PTR; 546 547 return kmem_cache_alloc_trace( 548 kmalloc_caches[kmalloc_type(flags)][index], 549 flags, size); 550 #endif 551 } 552 return __kmalloc(size, flags); 553 } 554 555 /* 556 * Determine size used for the nth kmalloc cache. 557 * return size or 0 if a kmalloc cache for that 558 * size does not exist 559 */ 560 static __always_inline unsigned int kmalloc_size(unsigned int n) 561 { 562 #ifndef CONFIG_SLOB 563 if (n > 2) 564 return 1U << n; 565 566 if (n == 1 && KMALLOC_MIN_SIZE <= 32) 567 return 96; 568 569 if (n == 2 && KMALLOC_MIN_SIZE <= 64) 570 return 192; 571 #endif 572 return 0; 573 } 574 575 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 576 { 577 #ifndef CONFIG_SLOB 578 if (__builtin_constant_p(size) && 579 size <= KMALLOC_MAX_CACHE_SIZE) { 580 unsigned int i = kmalloc_index(size); 581 582 if (!i) 583 return ZERO_SIZE_PTR; 584 585 return kmem_cache_alloc_node_trace( 586 kmalloc_caches[kmalloc_type(flags)][i], 587 flags, node, size); 588 } 589 #endif 590 return __kmalloc_node(size, flags, node); 591 } 592 593 struct memcg_cache_array { 594 struct rcu_head rcu; 595 struct kmem_cache *entries[0]; 596 }; 597 598 /* 599 * This is the main placeholder for memcg-related information in kmem caches. 600 * Both the root cache and the child caches will have it. For the root cache, 601 * this will hold a dynamically allocated array large enough to hold 602 * information about the currently limited memcgs in the system. To allow the 603 * array to be accessed without taking any locks, on relocation we free the old 604 * version only after a grace period. 605 * 606 * Root and child caches hold different metadata. 607 * 608 * @root_cache: Common to root and child caches. NULL for root, pointer to 609 * the root cache for children. 610 * 611 * The following fields are specific to root caches. 612 * 613 * @memcg_caches: kmemcg ID indexed table of child caches. This table is 614 * used to index child cachces during allocation and cleared 615 * early during shutdown. 616 * 617 * @root_caches_node: List node for slab_root_caches list. 618 * 619 * @children: List of all child caches. While the child caches are also 620 * reachable through @memcg_caches, a child cache remains on 621 * this list until it is actually destroyed. 622 * 623 * The following fields are specific to child caches. 624 * 625 * @memcg: Pointer to the memcg this cache belongs to. 626 * 627 * @children_node: List node for @root_cache->children list. 628 * 629 * @kmem_caches_node: List node for @memcg->kmem_caches list. 630 */ 631 struct memcg_cache_params { 632 struct kmem_cache *root_cache; 633 union { 634 struct { 635 struct memcg_cache_array __rcu *memcg_caches; 636 struct list_head __root_caches_node; 637 struct list_head children; 638 bool dying; 639 }; 640 struct { 641 struct mem_cgroup *memcg; 642 struct list_head children_node; 643 struct list_head kmem_caches_node; 644 645 void (*deact_fn)(struct kmem_cache *); 646 union { 647 struct rcu_head deact_rcu_head; 648 struct work_struct deact_work; 649 }; 650 }; 651 }; 652 }; 653 654 int memcg_update_all_caches(int num_memcgs); 655 656 /** 657 * kmalloc_array - allocate memory for an array. 658 * @n: number of elements. 659 * @size: element size. 660 * @flags: the type of memory to allocate (see kmalloc). 661 */ 662 static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags) 663 { 664 size_t bytes; 665 666 if (unlikely(check_mul_overflow(n, size, &bytes))) 667 return NULL; 668 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 669 return kmalloc(bytes, flags); 670 return __kmalloc(bytes, flags); 671 } 672 673 /** 674 * kcalloc - allocate memory for an array. The memory is set to zero. 675 * @n: number of elements. 676 * @size: element size. 677 * @flags: the type of memory to allocate (see kmalloc). 678 */ 679 static inline void *kcalloc(size_t n, size_t size, gfp_t flags) 680 { 681 return kmalloc_array(n, size, flags | __GFP_ZERO); 682 } 683 684 /* 685 * kmalloc_track_caller is a special version of kmalloc that records the 686 * calling function of the routine calling it for slab leak tracking instead 687 * of just the calling function (confusing, eh?). 688 * It's useful when the call to kmalloc comes from a widely-used standard 689 * allocator where we care about the real place the memory allocation 690 * request comes from. 691 */ 692 extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); 693 #define kmalloc_track_caller(size, flags) \ 694 __kmalloc_track_caller(size, flags, _RET_IP_) 695 696 static inline void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, 697 int node) 698 { 699 size_t bytes; 700 701 if (unlikely(check_mul_overflow(n, size, &bytes))) 702 return NULL; 703 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 704 return kmalloc_node(bytes, flags, node); 705 return __kmalloc_node(bytes, flags, node); 706 } 707 708 static inline void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) 709 { 710 return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); 711 } 712 713 714 #ifdef CONFIG_NUMA 715 extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long); 716 #define kmalloc_node_track_caller(size, flags, node) \ 717 __kmalloc_node_track_caller(size, flags, node, \ 718 _RET_IP_) 719 720 #else /* CONFIG_NUMA */ 721 722 #define kmalloc_node_track_caller(size, flags, node) \ 723 kmalloc_track_caller(size, flags) 724 725 #endif /* CONFIG_NUMA */ 726 727 /* 728 * Shortcuts 729 */ 730 static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) 731 { 732 return kmem_cache_alloc(k, flags | __GFP_ZERO); 733 } 734 735 /** 736 * kzalloc - allocate memory. The memory is set to zero. 737 * @size: how many bytes of memory are required. 738 * @flags: the type of memory to allocate (see kmalloc). 739 */ 740 static inline void *kzalloc(size_t size, gfp_t flags) 741 { 742 return kmalloc(size, flags | __GFP_ZERO); 743 } 744 745 /** 746 * kzalloc_node - allocate zeroed memory from a particular memory node. 747 * @size: how many bytes of memory are required. 748 * @flags: the type of memory to allocate (see kmalloc). 749 * @node: memory node from which to allocate 750 */ 751 static inline void *kzalloc_node(size_t size, gfp_t flags, int node) 752 { 753 return kmalloc_node(size, flags | __GFP_ZERO, node); 754 } 755 756 unsigned int kmem_cache_size(struct kmem_cache *s); 757 void __init kmem_cache_init_late(void); 758 759 #if defined(CONFIG_SMP) && defined(CONFIG_SLAB) 760 int slab_prepare_cpu(unsigned int cpu); 761 int slab_dead_cpu(unsigned int cpu); 762 #else 763 #define slab_prepare_cpu NULL 764 #define slab_dead_cpu NULL 765 #endif 766 767 #endif /* _LINUX_SLAB_H */ 768