1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Written by Mark Hemment, 1996 (markhe@nextd.demon.co.uk). 4 * 5 * (C) SGI 2006, Christoph Lameter 6 * Cleaned up and restructured to ease the addition of alternative 7 * implementations of SLAB allocators. 8 * (C) Linux Foundation 2008-2013 9 * Unified interface for all slab allocators 10 */ 11 12 #ifndef _LINUX_SLAB_H 13 #define _LINUX_SLAB_H 14 15 #include <linux/gfp.h> 16 #include <linux/overflow.h> 17 #include <linux/types.h> 18 #include <linux/workqueue.h> 19 20 21 /* 22 * Flags to pass to kmem_cache_create(). 23 * The ones marked DEBUG are only valid if CONFIG_DEBUG_SLAB is set. 24 */ 25 /* DEBUG: Perform (expensive) checks on alloc/free */ 26 #define SLAB_CONSISTENCY_CHECKS ((slab_flags_t __force)0x00000100U) 27 /* DEBUG: Red zone objs in a cache */ 28 #define SLAB_RED_ZONE ((slab_flags_t __force)0x00000400U) 29 /* DEBUG: Poison objects */ 30 #define SLAB_POISON ((slab_flags_t __force)0x00000800U) 31 /* Align objs on cache lines */ 32 #define SLAB_HWCACHE_ALIGN ((slab_flags_t __force)0x00002000U) 33 /* Use GFP_DMA memory */ 34 #define SLAB_CACHE_DMA ((slab_flags_t __force)0x00004000U) 35 /* DEBUG: Store the last owner for bug hunting */ 36 #define SLAB_STORE_USER ((slab_flags_t __force)0x00010000U) 37 /* Panic if kmem_cache_create() fails */ 38 #define SLAB_PANIC ((slab_flags_t __force)0x00040000U) 39 /* 40 * SLAB_TYPESAFE_BY_RCU - **WARNING** READ THIS! 41 * 42 * This delays freeing the SLAB page by a grace period, it does _NOT_ 43 * delay object freeing. This means that if you do kmem_cache_free() 44 * that memory location is free to be reused at any time. Thus it may 45 * be possible to see another object there in the same RCU grace period. 46 * 47 * This feature only ensures the memory location backing the object 48 * stays valid, the trick to using this is relying on an independent 49 * object validation pass. Something like: 50 * 51 * rcu_read_lock() 52 * again: 53 * obj = lockless_lookup(key); 54 * if (obj) { 55 * if (!try_get_ref(obj)) // might fail for free objects 56 * goto again; 57 * 58 * if (obj->key != key) { // not the object we expected 59 * put_ref(obj); 60 * goto again; 61 * } 62 * } 63 * rcu_read_unlock(); 64 * 65 * This is useful if we need to approach a kernel structure obliquely, 66 * from its address obtained without the usual locking. We can lock 67 * the structure to stabilize it and check it's still at the given address, 68 * only if we can be sure that the memory has not been meanwhile reused 69 * for some other kind of object (which our subsystem's lock might corrupt). 70 * 71 * rcu_read_lock before reading the address, then rcu_read_unlock after 72 * taking the spinlock within the structure expected at that address. 73 * 74 * Note that SLAB_TYPESAFE_BY_RCU was originally named SLAB_DESTROY_BY_RCU. 75 */ 76 /* Defer freeing slabs to RCU */ 77 #define SLAB_TYPESAFE_BY_RCU ((slab_flags_t __force)0x00080000U) 78 /* Spread some memory over cpuset */ 79 #define SLAB_MEM_SPREAD ((slab_flags_t __force)0x00100000U) 80 /* Trace allocations and frees */ 81 #define SLAB_TRACE ((slab_flags_t __force)0x00200000U) 82 83 /* Flag to prevent checks on free */ 84 #ifdef CONFIG_DEBUG_OBJECTS 85 # define SLAB_DEBUG_OBJECTS ((slab_flags_t __force)0x00400000U) 86 #else 87 # define SLAB_DEBUG_OBJECTS 0 88 #endif 89 90 /* Avoid kmemleak tracing */ 91 #define SLAB_NOLEAKTRACE ((slab_flags_t __force)0x00800000U) 92 93 /* Fault injection mark */ 94 #ifdef CONFIG_FAILSLAB 95 # define SLAB_FAILSLAB ((slab_flags_t __force)0x02000000U) 96 #else 97 # define SLAB_FAILSLAB 0 98 #endif 99 /* Account to memcg */ 100 #ifdef CONFIG_MEMCG_KMEM 101 # define SLAB_ACCOUNT ((slab_flags_t __force)0x04000000U) 102 #else 103 # define SLAB_ACCOUNT 0 104 #endif 105 106 #ifdef CONFIG_KASAN 107 #define SLAB_KASAN ((slab_flags_t __force)0x08000000U) 108 #else 109 #define SLAB_KASAN 0 110 #endif 111 112 /* The following flags affect the page allocator grouping pages by mobility */ 113 /* Objects are reclaimable */ 114 #define SLAB_RECLAIM_ACCOUNT ((slab_flags_t __force)0x00020000U) 115 #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ 116 /* 117 * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. 118 * 119 * Dereferencing ZERO_SIZE_PTR will lead to a distinct access fault. 120 * 121 * ZERO_SIZE_PTR can be passed to kfree though in the same way that NULL can. 122 * Both make kfree a no-op. 123 */ 124 #define ZERO_SIZE_PTR ((void *)16) 125 126 #define ZERO_OR_NULL_PTR(x) ((unsigned long)(x) <= \ 127 (unsigned long)ZERO_SIZE_PTR) 128 129 #include <linux/kasan.h> 130 131 struct mem_cgroup; 132 /* 133 * struct kmem_cache related prototypes 134 */ 135 void __init kmem_cache_init(void); 136 bool slab_is_available(void); 137 138 extern bool usercopy_fallback; 139 140 struct kmem_cache *kmem_cache_create(const char *name, unsigned int size, 141 unsigned int align, slab_flags_t flags, 142 void (*ctor)(void *)); 143 struct kmem_cache *kmem_cache_create_usercopy(const char *name, 144 unsigned int size, unsigned int align, 145 slab_flags_t flags, 146 unsigned int useroffset, unsigned int usersize, 147 void (*ctor)(void *)); 148 void kmem_cache_destroy(struct kmem_cache *); 149 int kmem_cache_shrink(struct kmem_cache *); 150 151 void memcg_create_kmem_cache(struct mem_cgroup *, struct kmem_cache *); 152 void memcg_deactivate_kmem_caches(struct mem_cgroup *); 153 void memcg_destroy_kmem_caches(struct mem_cgroup *); 154 155 /* 156 * Please use this macro to create slab caches. Simply specify the 157 * name of the structure and maybe some flags that are listed above. 158 * 159 * The alignment of the struct determines object alignment. If you 160 * f.e. add ____cacheline_aligned_in_smp to the struct declaration 161 * then the objects will be properly aligned in SMP configurations. 162 */ 163 #define KMEM_CACHE(__struct, __flags) \ 164 kmem_cache_create(#__struct, sizeof(struct __struct), \ 165 __alignof__(struct __struct), (__flags), NULL) 166 167 /* 168 * To whitelist a single field for copying to/from usercopy, use this 169 * macro instead for KMEM_CACHE() above. 170 */ 171 #define KMEM_CACHE_USERCOPY(__struct, __flags, __field) \ 172 kmem_cache_create_usercopy(#__struct, \ 173 sizeof(struct __struct), \ 174 __alignof__(struct __struct), (__flags), \ 175 offsetof(struct __struct, __field), \ 176 sizeof_field(struct __struct, __field), NULL) 177 178 /* 179 * Common kmalloc functions provided by all allocators 180 */ 181 void * __must_check __krealloc(const void *, size_t, gfp_t); 182 void * __must_check krealloc(const void *, size_t, gfp_t); 183 void kfree(const void *); 184 void kzfree(const void *); 185 size_t ksize(const void *); 186 187 #ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR 188 void __check_heap_object(const void *ptr, unsigned long n, struct page *page, 189 bool to_user); 190 #else 191 static inline void __check_heap_object(const void *ptr, unsigned long n, 192 struct page *page, bool to_user) { } 193 #endif 194 195 /* 196 * Some archs want to perform DMA into kmalloc caches and need a guaranteed 197 * alignment larger than the alignment of a 64-bit integer. 198 * Setting ARCH_KMALLOC_MINALIGN in arch headers allows that. 199 */ 200 #if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8 201 #define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN 202 #define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN 203 #define KMALLOC_SHIFT_LOW ilog2(ARCH_DMA_MINALIGN) 204 #else 205 #define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long) 206 #endif 207 208 /* 209 * Setting ARCH_SLAB_MINALIGN in arch headers allows a different alignment. 210 * Intended for arches that get misalignment faults even for 64 bit integer 211 * aligned buffers. 212 */ 213 #ifndef ARCH_SLAB_MINALIGN 214 #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 215 #endif 216 217 /* 218 * kmalloc and friends return ARCH_KMALLOC_MINALIGN aligned 219 * pointers. kmem_cache_alloc and friends return ARCH_SLAB_MINALIGN 220 * aligned pointers. 221 */ 222 #define __assume_kmalloc_alignment __assume_aligned(ARCH_KMALLOC_MINALIGN) 223 #define __assume_slab_alignment __assume_aligned(ARCH_SLAB_MINALIGN) 224 #define __assume_page_alignment __assume_aligned(PAGE_SIZE) 225 226 /* 227 * Kmalloc array related definitions 228 */ 229 230 #ifdef CONFIG_SLAB 231 /* 232 * The largest kmalloc size supported by the SLAB allocators is 233 * 32 megabyte (2^25) or the maximum allocatable page order if that is 234 * less than 32 MB. 235 * 236 * WARNING: Its not easy to increase this value since the allocators have 237 * to do various tricks to work around compiler limitations in order to 238 * ensure proper constant folding. 239 */ 240 #define KMALLOC_SHIFT_HIGH ((MAX_ORDER + PAGE_SHIFT - 1) <= 25 ? \ 241 (MAX_ORDER + PAGE_SHIFT - 1) : 25) 242 #define KMALLOC_SHIFT_MAX KMALLOC_SHIFT_HIGH 243 #ifndef KMALLOC_SHIFT_LOW 244 #define KMALLOC_SHIFT_LOW 5 245 #endif 246 #endif 247 248 #ifdef CONFIG_SLUB 249 /* 250 * SLUB directly allocates requests fitting in to an order-1 page 251 * (PAGE_SIZE*2). Larger requests are passed to the page allocator. 252 */ 253 #define KMALLOC_SHIFT_HIGH (PAGE_SHIFT + 1) 254 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 255 #ifndef KMALLOC_SHIFT_LOW 256 #define KMALLOC_SHIFT_LOW 3 257 #endif 258 #endif 259 260 #ifdef CONFIG_SLOB 261 /* 262 * SLOB passes all requests larger than one page to the page allocator. 263 * No kmalloc array is necessary since objects of different sizes can 264 * be allocated from the same page. 265 */ 266 #define KMALLOC_SHIFT_HIGH PAGE_SHIFT 267 #define KMALLOC_SHIFT_MAX (MAX_ORDER + PAGE_SHIFT - 1) 268 #ifndef KMALLOC_SHIFT_LOW 269 #define KMALLOC_SHIFT_LOW 3 270 #endif 271 #endif 272 273 /* Maximum allocatable size */ 274 #define KMALLOC_MAX_SIZE (1UL << KMALLOC_SHIFT_MAX) 275 /* Maximum size for which we actually use a slab cache */ 276 #define KMALLOC_MAX_CACHE_SIZE (1UL << KMALLOC_SHIFT_HIGH) 277 /* Maximum order allocatable via the slab allocagtor */ 278 #define KMALLOC_MAX_ORDER (KMALLOC_SHIFT_MAX - PAGE_SHIFT) 279 280 /* 281 * Kmalloc subsystem. 282 */ 283 #ifndef KMALLOC_MIN_SIZE 284 #define KMALLOC_MIN_SIZE (1 << KMALLOC_SHIFT_LOW) 285 #endif 286 287 /* 288 * This restriction comes from byte sized index implementation. 289 * Page size is normally 2^12 bytes and, in this case, if we want to use 290 * byte sized index which can represent 2^8 entries, the size of the object 291 * should be equal or greater to 2^12 / 2^8 = 2^4 = 16. 292 * If minimum size of kmalloc is less than 16, we use it as minimum object 293 * size and give up to use byte sized index. 294 */ 295 #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ 296 (KMALLOC_MIN_SIZE) : 16) 297 298 /* 299 * Whenever changing this, take care of that kmalloc_type() and 300 * create_kmalloc_caches() still work as intended. 301 */ 302 enum kmalloc_cache_type { 303 KMALLOC_NORMAL = 0, 304 KMALLOC_RECLAIM, 305 #ifdef CONFIG_ZONE_DMA 306 KMALLOC_DMA, 307 #endif 308 NR_KMALLOC_TYPES 309 }; 310 311 #ifndef CONFIG_SLOB 312 extern struct kmem_cache * 313 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1]; 314 315 static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags) 316 { 317 #ifdef CONFIG_ZONE_DMA 318 /* 319 * The most common case is KMALLOC_NORMAL, so test for it 320 * with a single branch for both flags. 321 */ 322 if (likely((flags & (__GFP_DMA | __GFP_RECLAIMABLE)) == 0)) 323 return KMALLOC_NORMAL; 324 325 /* 326 * At least one of the flags has to be set. If both are, __GFP_DMA 327 * is more important. 328 */ 329 return flags & __GFP_DMA ? KMALLOC_DMA : KMALLOC_RECLAIM; 330 #else 331 return flags & __GFP_RECLAIMABLE ? KMALLOC_RECLAIM : KMALLOC_NORMAL; 332 #endif 333 } 334 335 /* 336 * Figure out which kmalloc slab an allocation of a certain size 337 * belongs to. 338 * 0 = zero alloc 339 * 1 = 65 .. 96 bytes 340 * 2 = 129 .. 192 bytes 341 * n = 2^(n-1)+1 .. 2^n 342 */ 343 static __always_inline unsigned int kmalloc_index(size_t size) 344 { 345 if (!size) 346 return 0; 347 348 if (size <= KMALLOC_MIN_SIZE) 349 return KMALLOC_SHIFT_LOW; 350 351 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96) 352 return 1; 353 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192) 354 return 2; 355 if (size <= 8) return 3; 356 if (size <= 16) return 4; 357 if (size <= 32) return 5; 358 if (size <= 64) return 6; 359 if (size <= 128) return 7; 360 if (size <= 256) return 8; 361 if (size <= 512) return 9; 362 if (size <= 1024) return 10; 363 if (size <= 2 * 1024) return 11; 364 if (size <= 4 * 1024) return 12; 365 if (size <= 8 * 1024) return 13; 366 if (size <= 16 * 1024) return 14; 367 if (size <= 32 * 1024) return 15; 368 if (size <= 64 * 1024) return 16; 369 if (size <= 128 * 1024) return 17; 370 if (size <= 256 * 1024) return 18; 371 if (size <= 512 * 1024) return 19; 372 if (size <= 1024 * 1024) return 20; 373 if (size <= 2 * 1024 * 1024) return 21; 374 if (size <= 4 * 1024 * 1024) return 22; 375 if (size <= 8 * 1024 * 1024) return 23; 376 if (size <= 16 * 1024 * 1024) return 24; 377 if (size <= 32 * 1024 * 1024) return 25; 378 if (size <= 64 * 1024 * 1024) return 26; 379 BUG(); 380 381 /* Will never be reached. Needed because the compiler may complain */ 382 return -1; 383 } 384 #endif /* !CONFIG_SLOB */ 385 386 void *__kmalloc(size_t size, gfp_t flags) __assume_kmalloc_alignment __malloc; 387 void *kmem_cache_alloc(struct kmem_cache *, gfp_t flags) __assume_slab_alignment __malloc; 388 void kmem_cache_free(struct kmem_cache *, void *); 389 390 /* 391 * Bulk allocation and freeing operations. These are accelerated in an 392 * allocator specific way to avoid taking locks repeatedly or building 393 * metadata structures unnecessarily. 394 * 395 * Note that interrupts must be enabled when calling these functions. 396 */ 397 void kmem_cache_free_bulk(struct kmem_cache *, size_t, void **); 398 int kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **); 399 400 /* 401 * Caller must not use kfree_bulk() on memory not originally allocated 402 * by kmalloc(), because the SLOB allocator cannot handle this. 403 */ 404 static __always_inline void kfree_bulk(size_t size, void **p) 405 { 406 kmem_cache_free_bulk(NULL, size, p); 407 } 408 409 #ifdef CONFIG_NUMA 410 void *__kmalloc_node(size_t size, gfp_t flags, int node) __assume_kmalloc_alignment __malloc; 411 void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node) __assume_slab_alignment __malloc; 412 #else 413 static __always_inline void *__kmalloc_node(size_t size, gfp_t flags, int node) 414 { 415 return __kmalloc(size, flags); 416 } 417 418 static __always_inline void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t flags, int node) 419 { 420 return kmem_cache_alloc(s, flags); 421 } 422 #endif 423 424 #ifdef CONFIG_TRACING 425 extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t) __assume_slab_alignment __malloc; 426 427 #ifdef CONFIG_NUMA 428 extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 429 gfp_t gfpflags, 430 int node, size_t size) __assume_slab_alignment __malloc; 431 #else 432 static __always_inline void * 433 kmem_cache_alloc_node_trace(struct kmem_cache *s, 434 gfp_t gfpflags, 435 int node, size_t size) 436 { 437 return kmem_cache_alloc_trace(s, gfpflags, size); 438 } 439 #endif /* CONFIG_NUMA */ 440 441 #else /* CONFIG_TRACING */ 442 static __always_inline void *kmem_cache_alloc_trace(struct kmem_cache *s, 443 gfp_t flags, size_t size) 444 { 445 void *ret = kmem_cache_alloc(s, flags); 446 447 ret = kasan_kmalloc(s, ret, size, flags); 448 return ret; 449 } 450 451 static __always_inline void * 452 kmem_cache_alloc_node_trace(struct kmem_cache *s, 453 gfp_t gfpflags, 454 int node, size_t size) 455 { 456 void *ret = kmem_cache_alloc_node(s, gfpflags, node); 457 458 ret = kasan_kmalloc(s, ret, size, gfpflags); 459 return ret; 460 } 461 #endif /* CONFIG_TRACING */ 462 463 extern void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; 464 465 #ifdef CONFIG_TRACING 466 extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) __assume_page_alignment __malloc; 467 #else 468 static __always_inline void * 469 kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 470 { 471 return kmalloc_order(size, flags, order); 472 } 473 #endif 474 475 static __always_inline void *kmalloc_large(size_t size, gfp_t flags) 476 { 477 unsigned int order = get_order(size); 478 return kmalloc_order_trace(size, flags, order); 479 } 480 481 /** 482 * kmalloc - allocate memory 483 * @size: how many bytes of memory are required. 484 * @flags: the type of memory to allocate. 485 * 486 * kmalloc is the normal method of allocating memory 487 * for objects smaller than page size in the kernel. 488 * 489 * The @flags argument may be one of the GFP flags defined at 490 * include/linux/gfp.h and described at 491 * :ref:`Documentation/core-api/mm-api.rst <mm-api-gfp-flags>` 492 * 493 * The recommended usage of the @flags is described at 494 * :ref:`Documentation/core-api/memory-allocation.rst <memory-allocation>` 495 * 496 * Below is a brief outline of the most useful GFP flags 497 * 498 * %GFP_KERNEL 499 * Allocate normal kernel ram. May sleep. 500 * 501 * %GFP_NOWAIT 502 * Allocation will not sleep. 503 * 504 * %GFP_ATOMIC 505 * Allocation will not sleep. May use emergency pools. 506 * 507 * %GFP_HIGHUSER 508 * Allocate memory from high memory on behalf of user. 509 * 510 * Also it is possible to set different flags by OR'ing 511 * in one or more of the following additional @flags: 512 * 513 * %__GFP_HIGH 514 * This allocation has high priority and may use emergency pools. 515 * 516 * %__GFP_NOFAIL 517 * Indicate that this allocation is in no way allowed to fail 518 * (think twice before using). 519 * 520 * %__GFP_NORETRY 521 * If memory is not immediately available, 522 * then give up at once. 523 * 524 * %__GFP_NOWARN 525 * If allocation fails, don't issue any warnings. 526 * 527 * %__GFP_RETRY_MAYFAIL 528 * Try really hard to succeed the allocation but fail 529 * eventually. 530 */ 531 static __always_inline void *kmalloc(size_t size, gfp_t flags) 532 { 533 if (__builtin_constant_p(size)) { 534 #ifndef CONFIG_SLOB 535 unsigned int index; 536 #endif 537 if (size > KMALLOC_MAX_CACHE_SIZE) 538 return kmalloc_large(size, flags); 539 #ifndef CONFIG_SLOB 540 index = kmalloc_index(size); 541 542 if (!index) 543 return ZERO_SIZE_PTR; 544 545 return kmem_cache_alloc_trace( 546 kmalloc_caches[kmalloc_type(flags)][index], 547 flags, size); 548 #endif 549 } 550 return __kmalloc(size, flags); 551 } 552 553 /* 554 * Determine size used for the nth kmalloc cache. 555 * return size or 0 if a kmalloc cache for that 556 * size does not exist 557 */ 558 static __always_inline unsigned int kmalloc_size(unsigned int n) 559 { 560 #ifndef CONFIG_SLOB 561 if (n > 2) 562 return 1U << n; 563 564 if (n == 1 && KMALLOC_MIN_SIZE <= 32) 565 return 96; 566 567 if (n == 2 && KMALLOC_MIN_SIZE <= 64) 568 return 192; 569 #endif 570 return 0; 571 } 572 573 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) 574 { 575 #ifndef CONFIG_SLOB 576 if (__builtin_constant_p(size) && 577 size <= KMALLOC_MAX_CACHE_SIZE) { 578 unsigned int i = kmalloc_index(size); 579 580 if (!i) 581 return ZERO_SIZE_PTR; 582 583 return kmem_cache_alloc_node_trace( 584 kmalloc_caches[kmalloc_type(flags)][i], 585 flags, node, size); 586 } 587 #endif 588 return __kmalloc_node(size, flags, node); 589 } 590 591 struct memcg_cache_array { 592 struct rcu_head rcu; 593 struct kmem_cache *entries[0]; 594 }; 595 596 /* 597 * This is the main placeholder for memcg-related information in kmem caches. 598 * Both the root cache and the child caches will have it. For the root cache, 599 * this will hold a dynamically allocated array large enough to hold 600 * information about the currently limited memcgs in the system. To allow the 601 * array to be accessed without taking any locks, on relocation we free the old 602 * version only after a grace period. 603 * 604 * Root and child caches hold different metadata. 605 * 606 * @root_cache: Common to root and child caches. NULL for root, pointer to 607 * the root cache for children. 608 * 609 * The following fields are specific to root caches. 610 * 611 * @memcg_caches: kmemcg ID indexed table of child caches. This table is 612 * used to index child cachces during allocation and cleared 613 * early during shutdown. 614 * 615 * @root_caches_node: List node for slab_root_caches list. 616 * 617 * @children: List of all child caches. While the child caches are also 618 * reachable through @memcg_caches, a child cache remains on 619 * this list until it is actually destroyed. 620 * 621 * The following fields are specific to child caches. 622 * 623 * @memcg: Pointer to the memcg this cache belongs to. 624 * 625 * @children_node: List node for @root_cache->children list. 626 * 627 * @kmem_caches_node: List node for @memcg->kmem_caches list. 628 */ 629 struct memcg_cache_params { 630 struct kmem_cache *root_cache; 631 union { 632 struct { 633 struct memcg_cache_array __rcu *memcg_caches; 634 struct list_head __root_caches_node; 635 struct list_head children; 636 bool dying; 637 }; 638 struct { 639 struct mem_cgroup *memcg; 640 struct list_head children_node; 641 struct list_head kmem_caches_node; 642 643 void (*deact_fn)(struct kmem_cache *); 644 union { 645 struct rcu_head deact_rcu_head; 646 struct work_struct deact_work; 647 }; 648 }; 649 }; 650 }; 651 652 int memcg_update_all_caches(int num_memcgs); 653 654 /** 655 * kmalloc_array - allocate memory for an array. 656 * @n: number of elements. 657 * @size: element size. 658 * @flags: the type of memory to allocate (see kmalloc). 659 */ 660 static inline void *kmalloc_array(size_t n, size_t size, gfp_t flags) 661 { 662 size_t bytes; 663 664 if (unlikely(check_mul_overflow(n, size, &bytes))) 665 return NULL; 666 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 667 return kmalloc(bytes, flags); 668 return __kmalloc(bytes, flags); 669 } 670 671 /** 672 * kcalloc - allocate memory for an array. The memory is set to zero. 673 * @n: number of elements. 674 * @size: element size. 675 * @flags: the type of memory to allocate (see kmalloc). 676 */ 677 static inline void *kcalloc(size_t n, size_t size, gfp_t flags) 678 { 679 return kmalloc_array(n, size, flags | __GFP_ZERO); 680 } 681 682 /* 683 * kmalloc_track_caller is a special version of kmalloc that records the 684 * calling function of the routine calling it for slab leak tracking instead 685 * of just the calling function (confusing, eh?). 686 * It's useful when the call to kmalloc comes from a widely-used standard 687 * allocator where we care about the real place the memory allocation 688 * request comes from. 689 */ 690 extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); 691 #define kmalloc_track_caller(size, flags) \ 692 __kmalloc_track_caller(size, flags, _RET_IP_) 693 694 static inline void *kmalloc_array_node(size_t n, size_t size, gfp_t flags, 695 int node) 696 { 697 size_t bytes; 698 699 if (unlikely(check_mul_overflow(n, size, &bytes))) 700 return NULL; 701 if (__builtin_constant_p(n) && __builtin_constant_p(size)) 702 return kmalloc_node(bytes, flags, node); 703 return __kmalloc_node(bytes, flags, node); 704 } 705 706 static inline void *kcalloc_node(size_t n, size_t size, gfp_t flags, int node) 707 { 708 return kmalloc_array_node(n, size, flags | __GFP_ZERO, node); 709 } 710 711 712 #ifdef CONFIG_NUMA 713 extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long); 714 #define kmalloc_node_track_caller(size, flags, node) \ 715 __kmalloc_node_track_caller(size, flags, node, \ 716 _RET_IP_) 717 718 #else /* CONFIG_NUMA */ 719 720 #define kmalloc_node_track_caller(size, flags, node) \ 721 kmalloc_track_caller(size, flags) 722 723 #endif /* CONFIG_NUMA */ 724 725 /* 726 * Shortcuts 727 */ 728 static inline void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) 729 { 730 return kmem_cache_alloc(k, flags | __GFP_ZERO); 731 } 732 733 /** 734 * kzalloc - allocate memory. The memory is set to zero. 735 * @size: how many bytes of memory are required. 736 * @flags: the type of memory to allocate (see kmalloc). 737 */ 738 static inline void *kzalloc(size_t size, gfp_t flags) 739 { 740 return kmalloc(size, flags | __GFP_ZERO); 741 } 742 743 /** 744 * kzalloc_node - allocate zeroed memory from a particular memory node. 745 * @size: how many bytes of memory are required. 746 * @flags: the type of memory to allocate (see kmalloc). 747 * @node: memory node from which to allocate 748 */ 749 static inline void *kzalloc_node(size_t size, gfp_t flags, int node) 750 { 751 return kmalloc_node(size, flags | __GFP_ZERO, node); 752 } 753 754 unsigned int kmem_cache_size(struct kmem_cache *s); 755 void __init kmem_cache_init_late(void); 756 757 #if defined(CONFIG_SMP) && defined(CONFIG_SLAB) 758 int slab_prepare_cpu(unsigned int cpu); 759 int slab_dead_cpu(unsigned int cpu); 760 #else 761 #define slab_prepare_cpu NULL 762 #define slab_dead_cpu NULL 763 #endif 764 765 #endif /* _LINUX_SLAB_H */ 766