1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Slab allocator functions that are independent of the allocator strategy 4 * 5 * (C) 2012 Christoph Lameter <cl@linux.com> 6 */ 7 #include <linux/slab.h> 8 9 #include <linux/mm.h> 10 #include <linux/poison.h> 11 #include <linux/interrupt.h> 12 #include <linux/memory.h> 13 #include <linux/cache.h> 14 #include <linux/compiler.h> 15 #include <linux/module.h> 16 #include <linux/cpu.h> 17 #include <linux/uaccess.h> 18 #include <linux/seq_file.h> 19 #include <linux/proc_fs.h> 20 #include <asm/cacheflush.h> 21 #include <asm/tlbflush.h> 22 #include <asm/page.h> 23 #include <linux/memcontrol.h> 24 25 #define CREATE_TRACE_POINTS 26 #include <trace/events/kmem.h> 27 28 #include "slab.h" 29 30 enum slab_state slab_state; 31 LIST_HEAD(slab_caches); 32 DEFINE_MUTEX(slab_mutex); 33 struct kmem_cache *kmem_cache; 34 35 #ifdef CONFIG_HARDENED_USERCOPY 36 bool usercopy_fallback __ro_after_init = 37 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK); 38 module_param(usercopy_fallback, bool, 0400); 39 MODULE_PARM_DESC(usercopy_fallback, 40 "WARN instead of reject usercopy whitelist violations"); 41 #endif 42 43 static LIST_HEAD(slab_caches_to_rcu_destroy); 44 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work); 45 static DECLARE_WORK(slab_caches_to_rcu_destroy_work, 46 slab_caches_to_rcu_destroy_workfn); 47 48 /* 49 * Set of flags that will prevent slab merging 50 */ 51 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 52 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ 53 SLAB_FAILSLAB | SLAB_KASAN) 54 55 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ 56 SLAB_ACCOUNT) 57 58 /* 59 * Merge control. If this is set then no merging of slab caches will occur. 60 */ 61 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT); 62 63 static int __init setup_slab_nomerge(char *str) 64 { 65 slab_nomerge = true; 66 return 1; 67 } 68 69 #ifdef CONFIG_SLUB 70 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0); 71 #endif 72 73 __setup("slab_nomerge", setup_slab_nomerge); 74 75 /* 76 * Determine the size of a slab object 77 */ 78 unsigned int kmem_cache_size(struct kmem_cache *s) 79 { 80 return s->object_size; 81 } 82 EXPORT_SYMBOL(kmem_cache_size); 83 84 #ifdef CONFIG_DEBUG_VM 85 static int kmem_cache_sanity_check(const char *name, unsigned int size) 86 { 87 if (!name || in_interrupt() || size < sizeof(void *) || 88 size > KMALLOC_MAX_SIZE) { 89 pr_err("kmem_cache_create(%s) integrity check failed\n", name); 90 return -EINVAL; 91 } 92 93 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 94 return 0; 95 } 96 #else 97 static inline int kmem_cache_sanity_check(const char *name, unsigned int size) 98 { 99 return 0; 100 } 101 #endif 102 103 void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 104 { 105 size_t i; 106 107 for (i = 0; i < nr; i++) { 108 if (s) 109 kmem_cache_free(s, p[i]); 110 else 111 kfree(p[i]); 112 } 113 } 114 115 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 116 void **p) 117 { 118 size_t i; 119 120 for (i = 0; i < nr; i++) { 121 void *x = p[i] = kmem_cache_alloc(s, flags); 122 if (!x) { 123 __kmem_cache_free_bulk(s, i, p); 124 return 0; 125 } 126 } 127 return i; 128 } 129 130 #ifdef CONFIG_MEMCG_KMEM 131 132 LIST_HEAD(slab_root_caches); 133 134 void slab_init_memcg_params(struct kmem_cache *s) 135 { 136 s->memcg_params.root_cache = NULL; 137 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); 138 INIT_LIST_HEAD(&s->memcg_params.children); 139 s->memcg_params.dying = false; 140 } 141 142 static int init_memcg_params(struct kmem_cache *s, 143 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 144 { 145 struct memcg_cache_array *arr; 146 147 if (root_cache) { 148 s->memcg_params.root_cache = root_cache; 149 s->memcg_params.memcg = memcg; 150 INIT_LIST_HEAD(&s->memcg_params.children_node); 151 INIT_LIST_HEAD(&s->memcg_params.kmem_caches_node); 152 return 0; 153 } 154 155 slab_init_memcg_params(s); 156 157 if (!memcg_nr_cache_ids) 158 return 0; 159 160 arr = kvzalloc(sizeof(struct memcg_cache_array) + 161 memcg_nr_cache_ids * sizeof(void *), 162 GFP_KERNEL); 163 if (!arr) 164 return -ENOMEM; 165 166 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr); 167 return 0; 168 } 169 170 static void destroy_memcg_params(struct kmem_cache *s) 171 { 172 if (is_root_cache(s)) 173 kvfree(rcu_access_pointer(s->memcg_params.memcg_caches)); 174 } 175 176 static void free_memcg_params(struct rcu_head *rcu) 177 { 178 struct memcg_cache_array *old; 179 180 old = container_of(rcu, struct memcg_cache_array, rcu); 181 kvfree(old); 182 } 183 184 static int update_memcg_params(struct kmem_cache *s, int new_array_size) 185 { 186 struct memcg_cache_array *old, *new; 187 188 new = kvzalloc(sizeof(struct memcg_cache_array) + 189 new_array_size * sizeof(void *), GFP_KERNEL); 190 if (!new) 191 return -ENOMEM; 192 193 old = rcu_dereference_protected(s->memcg_params.memcg_caches, 194 lockdep_is_held(&slab_mutex)); 195 if (old) 196 memcpy(new->entries, old->entries, 197 memcg_nr_cache_ids * sizeof(void *)); 198 199 rcu_assign_pointer(s->memcg_params.memcg_caches, new); 200 if (old) 201 call_rcu(&old->rcu, free_memcg_params); 202 return 0; 203 } 204 205 int memcg_update_all_caches(int num_memcgs) 206 { 207 struct kmem_cache *s; 208 int ret = 0; 209 210 mutex_lock(&slab_mutex); 211 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 212 ret = update_memcg_params(s, num_memcgs); 213 /* 214 * Instead of freeing the memory, we'll just leave the caches 215 * up to this point in an updated state. 216 */ 217 if (ret) 218 break; 219 } 220 mutex_unlock(&slab_mutex); 221 return ret; 222 } 223 224 void memcg_link_cache(struct kmem_cache *s) 225 { 226 if (is_root_cache(s)) { 227 list_add(&s->root_caches_node, &slab_root_caches); 228 } else { 229 list_add(&s->memcg_params.children_node, 230 &s->memcg_params.root_cache->memcg_params.children); 231 list_add(&s->memcg_params.kmem_caches_node, 232 &s->memcg_params.memcg->kmem_caches); 233 } 234 } 235 236 static void memcg_unlink_cache(struct kmem_cache *s) 237 { 238 if (is_root_cache(s)) { 239 list_del(&s->root_caches_node); 240 } else { 241 list_del(&s->memcg_params.children_node); 242 list_del(&s->memcg_params.kmem_caches_node); 243 } 244 } 245 #else 246 static inline int init_memcg_params(struct kmem_cache *s, 247 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 248 { 249 return 0; 250 } 251 252 static inline void destroy_memcg_params(struct kmem_cache *s) 253 { 254 } 255 256 static inline void memcg_unlink_cache(struct kmem_cache *s) 257 { 258 } 259 #endif /* CONFIG_MEMCG_KMEM */ 260 261 /* 262 * Figure out what the alignment of the objects will be given a set of 263 * flags, a user specified alignment and the size of the objects. 264 */ 265 static unsigned int calculate_alignment(slab_flags_t flags, 266 unsigned int align, unsigned int size) 267 { 268 /* 269 * If the user wants hardware cache aligned objects then follow that 270 * suggestion if the object is sufficiently large. 271 * 272 * The hardware cache alignment cannot override the specified 273 * alignment though. If that is greater then use it. 274 */ 275 if (flags & SLAB_HWCACHE_ALIGN) { 276 unsigned int ralign; 277 278 ralign = cache_line_size(); 279 while (size <= ralign / 2) 280 ralign /= 2; 281 align = max(align, ralign); 282 } 283 284 if (align < ARCH_SLAB_MINALIGN) 285 align = ARCH_SLAB_MINALIGN; 286 287 return ALIGN(align, sizeof(void *)); 288 } 289 290 /* 291 * Find a mergeable slab cache 292 */ 293 int slab_unmergeable(struct kmem_cache *s) 294 { 295 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) 296 return 1; 297 298 if (!is_root_cache(s)) 299 return 1; 300 301 if (s->ctor) 302 return 1; 303 304 if (s->usersize) 305 return 1; 306 307 /* 308 * We may have set a slab to be unmergeable during bootstrap. 309 */ 310 if (s->refcount < 0) 311 return 1; 312 313 return 0; 314 } 315 316 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, 317 slab_flags_t flags, const char *name, void (*ctor)(void *)) 318 { 319 struct kmem_cache *s; 320 321 if (slab_nomerge) 322 return NULL; 323 324 if (ctor) 325 return NULL; 326 327 size = ALIGN(size, sizeof(void *)); 328 align = calculate_alignment(flags, align, size); 329 size = ALIGN(size, align); 330 flags = kmem_cache_flags(size, flags, name, NULL); 331 332 if (flags & SLAB_NEVER_MERGE) 333 return NULL; 334 335 list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) { 336 if (slab_unmergeable(s)) 337 continue; 338 339 if (size > s->size) 340 continue; 341 342 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) 343 continue; 344 /* 345 * Check if alignment is compatible. 346 * Courtesy of Adrian Drzewiecki 347 */ 348 if ((s->size & ~(align - 1)) != s->size) 349 continue; 350 351 if (s->size - size >= sizeof(void *)) 352 continue; 353 354 if (IS_ENABLED(CONFIG_SLAB) && align && 355 (align > s->align || s->align % align)) 356 continue; 357 358 return s; 359 } 360 return NULL; 361 } 362 363 static struct kmem_cache *create_cache(const char *name, 364 unsigned int object_size, unsigned int align, 365 slab_flags_t flags, unsigned int useroffset, 366 unsigned int usersize, void (*ctor)(void *), 367 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 368 { 369 struct kmem_cache *s; 370 int err; 371 372 if (WARN_ON(useroffset + usersize > object_size)) 373 useroffset = usersize = 0; 374 375 err = -ENOMEM; 376 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 377 if (!s) 378 goto out; 379 380 s->name = name; 381 s->size = s->object_size = object_size; 382 s->align = align; 383 s->ctor = ctor; 384 s->useroffset = useroffset; 385 s->usersize = usersize; 386 387 err = init_memcg_params(s, memcg, root_cache); 388 if (err) 389 goto out_free_cache; 390 391 err = __kmem_cache_create(s, flags); 392 if (err) 393 goto out_free_cache; 394 395 s->refcount = 1; 396 list_add(&s->list, &slab_caches); 397 memcg_link_cache(s); 398 out: 399 if (err) 400 return ERR_PTR(err); 401 return s; 402 403 out_free_cache: 404 destroy_memcg_params(s); 405 kmem_cache_free(kmem_cache, s); 406 goto out; 407 } 408 409 /* 410 * kmem_cache_create_usercopy - Create a cache. 411 * @name: A string which is used in /proc/slabinfo to identify this cache. 412 * @size: The size of objects to be created in this cache. 413 * @align: The required alignment for the objects. 414 * @flags: SLAB flags 415 * @useroffset: Usercopy region offset 416 * @usersize: Usercopy region size 417 * @ctor: A constructor for the objects. 418 * 419 * Returns a ptr to the cache on success, NULL on failure. 420 * Cannot be called within a interrupt, but can be interrupted. 421 * The @ctor is run when new pages are allocated by the cache. 422 * 423 * The flags are 424 * 425 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 426 * to catch references to uninitialised memory. 427 * 428 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 429 * for buffer overruns. 430 * 431 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 432 * cacheline. This can be beneficial if you're counting cycles as closely 433 * as davem. 434 */ 435 struct kmem_cache * 436 kmem_cache_create_usercopy(const char *name, 437 unsigned int size, unsigned int align, 438 slab_flags_t flags, 439 unsigned int useroffset, unsigned int usersize, 440 void (*ctor)(void *)) 441 { 442 struct kmem_cache *s = NULL; 443 const char *cache_name; 444 int err; 445 446 get_online_cpus(); 447 get_online_mems(); 448 memcg_get_cache_ids(); 449 450 mutex_lock(&slab_mutex); 451 452 err = kmem_cache_sanity_check(name, size); 453 if (err) { 454 goto out_unlock; 455 } 456 457 /* Refuse requests with allocator specific flags */ 458 if (flags & ~SLAB_FLAGS_PERMITTED) { 459 err = -EINVAL; 460 goto out_unlock; 461 } 462 463 /* 464 * Some allocators will constraint the set of valid flags to a subset 465 * of all flags. We expect them to define CACHE_CREATE_MASK in this 466 * case, and we'll just provide them with a sanitized version of the 467 * passed flags. 468 */ 469 flags &= CACHE_CREATE_MASK; 470 471 /* Fail closed on bad usersize of useroffset values. */ 472 if (WARN_ON(!usersize && useroffset) || 473 WARN_ON(size < usersize || size - usersize < useroffset)) 474 usersize = useroffset = 0; 475 476 if (!usersize) 477 s = __kmem_cache_alias(name, size, align, flags, ctor); 478 if (s) 479 goto out_unlock; 480 481 cache_name = kstrdup_const(name, GFP_KERNEL); 482 if (!cache_name) { 483 err = -ENOMEM; 484 goto out_unlock; 485 } 486 487 s = create_cache(cache_name, size, 488 calculate_alignment(flags, align, size), 489 flags, useroffset, usersize, ctor, NULL, NULL); 490 if (IS_ERR(s)) { 491 err = PTR_ERR(s); 492 kfree_const(cache_name); 493 } 494 495 out_unlock: 496 mutex_unlock(&slab_mutex); 497 498 memcg_put_cache_ids(); 499 put_online_mems(); 500 put_online_cpus(); 501 502 if (err) { 503 if (flags & SLAB_PANIC) 504 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", 505 name, err); 506 else { 507 pr_warn("kmem_cache_create(%s) failed with error %d\n", 508 name, err); 509 dump_stack(); 510 } 511 return NULL; 512 } 513 return s; 514 } 515 EXPORT_SYMBOL(kmem_cache_create_usercopy); 516 517 struct kmem_cache * 518 kmem_cache_create(const char *name, unsigned int size, unsigned int align, 519 slab_flags_t flags, void (*ctor)(void *)) 520 { 521 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0, 522 ctor); 523 } 524 EXPORT_SYMBOL(kmem_cache_create); 525 526 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) 527 { 528 LIST_HEAD(to_destroy); 529 struct kmem_cache *s, *s2; 530 531 /* 532 * On destruction, SLAB_TYPESAFE_BY_RCU kmem_caches are put on the 533 * @slab_caches_to_rcu_destroy list. The slab pages are freed 534 * through RCU and and the associated kmem_cache are dereferenced 535 * while freeing the pages, so the kmem_caches should be freed only 536 * after the pending RCU operations are finished. As rcu_barrier() 537 * is a pretty slow operation, we batch all pending destructions 538 * asynchronously. 539 */ 540 mutex_lock(&slab_mutex); 541 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy); 542 mutex_unlock(&slab_mutex); 543 544 if (list_empty(&to_destroy)) 545 return; 546 547 rcu_barrier(); 548 549 list_for_each_entry_safe(s, s2, &to_destroy, list) { 550 #ifdef SLAB_SUPPORTS_SYSFS 551 sysfs_slab_release(s); 552 #else 553 slab_kmem_cache_release(s); 554 #endif 555 } 556 } 557 558 static int shutdown_cache(struct kmem_cache *s) 559 { 560 /* free asan quarantined objects */ 561 kasan_cache_shutdown(s); 562 563 if (__kmem_cache_shutdown(s) != 0) 564 return -EBUSY; 565 566 memcg_unlink_cache(s); 567 list_del(&s->list); 568 569 if (s->flags & SLAB_TYPESAFE_BY_RCU) { 570 #ifdef SLAB_SUPPORTS_SYSFS 571 sysfs_slab_unlink(s); 572 #endif 573 list_add_tail(&s->list, &slab_caches_to_rcu_destroy); 574 schedule_work(&slab_caches_to_rcu_destroy_work); 575 } else { 576 #ifdef SLAB_SUPPORTS_SYSFS 577 sysfs_slab_unlink(s); 578 sysfs_slab_release(s); 579 #else 580 slab_kmem_cache_release(s); 581 #endif 582 } 583 584 return 0; 585 } 586 587 #ifdef CONFIG_MEMCG_KMEM 588 /* 589 * memcg_create_kmem_cache - Create a cache for a memory cgroup. 590 * @memcg: The memory cgroup the new cache is for. 591 * @root_cache: The parent of the new cache. 592 * 593 * This function attempts to create a kmem cache that will serve allocation 594 * requests going from @memcg to @root_cache. The new cache inherits properties 595 * from its parent. 596 */ 597 void memcg_create_kmem_cache(struct mem_cgroup *memcg, 598 struct kmem_cache *root_cache) 599 { 600 static char memcg_name_buf[NAME_MAX + 1]; /* protected by slab_mutex */ 601 struct cgroup_subsys_state *css = &memcg->css; 602 struct memcg_cache_array *arr; 603 struct kmem_cache *s = NULL; 604 char *cache_name; 605 int idx; 606 607 get_online_cpus(); 608 get_online_mems(); 609 610 mutex_lock(&slab_mutex); 611 612 /* 613 * The memory cgroup could have been offlined while the cache 614 * creation work was pending. 615 */ 616 if (memcg->kmem_state != KMEM_ONLINE || root_cache->memcg_params.dying) 617 goto out_unlock; 618 619 idx = memcg_cache_id(memcg); 620 arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches, 621 lockdep_is_held(&slab_mutex)); 622 623 /* 624 * Since per-memcg caches are created asynchronously on first 625 * allocation (see memcg_kmem_get_cache()), several threads can try to 626 * create the same cache, but only one of them may succeed. 627 */ 628 if (arr->entries[idx]) 629 goto out_unlock; 630 631 cgroup_name(css->cgroup, memcg_name_buf, sizeof(memcg_name_buf)); 632 cache_name = kasprintf(GFP_KERNEL, "%s(%llu:%s)", root_cache->name, 633 css->serial_nr, memcg_name_buf); 634 if (!cache_name) 635 goto out_unlock; 636 637 s = create_cache(cache_name, root_cache->object_size, 638 root_cache->align, 639 root_cache->flags & CACHE_CREATE_MASK, 640 root_cache->useroffset, root_cache->usersize, 641 root_cache->ctor, memcg, root_cache); 642 /* 643 * If we could not create a memcg cache, do not complain, because 644 * that's not critical at all as we can always proceed with the root 645 * cache. 646 */ 647 if (IS_ERR(s)) { 648 kfree(cache_name); 649 goto out_unlock; 650 } 651 652 /* 653 * Since readers won't lock (see cache_from_memcg_idx()), we need a 654 * barrier here to ensure nobody will see the kmem_cache partially 655 * initialized. 656 */ 657 smp_wmb(); 658 arr->entries[idx] = s; 659 660 out_unlock: 661 mutex_unlock(&slab_mutex); 662 663 put_online_mems(); 664 put_online_cpus(); 665 } 666 667 static void kmemcg_deactivate_workfn(struct work_struct *work) 668 { 669 struct kmem_cache *s = container_of(work, struct kmem_cache, 670 memcg_params.deact_work); 671 672 get_online_cpus(); 673 get_online_mems(); 674 675 mutex_lock(&slab_mutex); 676 677 s->memcg_params.deact_fn(s); 678 679 mutex_unlock(&slab_mutex); 680 681 put_online_mems(); 682 put_online_cpus(); 683 684 /* done, put the ref from slab_deactivate_memcg_cache_rcu_sched() */ 685 css_put(&s->memcg_params.memcg->css); 686 } 687 688 static void kmemcg_deactivate_rcufn(struct rcu_head *head) 689 { 690 struct kmem_cache *s = container_of(head, struct kmem_cache, 691 memcg_params.deact_rcu_head); 692 693 /* 694 * We need to grab blocking locks. Bounce to ->deact_work. The 695 * work item shares the space with the RCU head and can't be 696 * initialized eariler. 697 */ 698 INIT_WORK(&s->memcg_params.deact_work, kmemcg_deactivate_workfn); 699 queue_work(memcg_kmem_cache_wq, &s->memcg_params.deact_work); 700 } 701 702 /** 703 * slab_deactivate_memcg_cache_rcu_sched - schedule deactivation after a 704 * sched RCU grace period 705 * @s: target kmem_cache 706 * @deact_fn: deactivation function to call 707 * 708 * Schedule @deact_fn to be invoked with online cpus, mems and slab_mutex 709 * held after a sched RCU grace period. The slab is guaranteed to stay 710 * alive until @deact_fn is finished. This is to be used from 711 * __kmemcg_cache_deactivate(). 712 */ 713 void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s, 714 void (*deact_fn)(struct kmem_cache *)) 715 { 716 if (WARN_ON_ONCE(is_root_cache(s)) || 717 WARN_ON_ONCE(s->memcg_params.deact_fn)) 718 return; 719 720 if (s->memcg_params.root_cache->memcg_params.dying) 721 return; 722 723 /* pin memcg so that @s doesn't get destroyed in the middle */ 724 css_get(&s->memcg_params.memcg->css); 725 726 s->memcg_params.deact_fn = deact_fn; 727 call_rcu_sched(&s->memcg_params.deact_rcu_head, kmemcg_deactivate_rcufn); 728 } 729 730 void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg) 731 { 732 int idx; 733 struct memcg_cache_array *arr; 734 struct kmem_cache *s, *c; 735 736 idx = memcg_cache_id(memcg); 737 738 get_online_cpus(); 739 get_online_mems(); 740 741 mutex_lock(&slab_mutex); 742 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 743 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 744 lockdep_is_held(&slab_mutex)); 745 c = arr->entries[idx]; 746 if (!c) 747 continue; 748 749 __kmemcg_cache_deactivate(c); 750 arr->entries[idx] = NULL; 751 } 752 mutex_unlock(&slab_mutex); 753 754 put_online_mems(); 755 put_online_cpus(); 756 } 757 758 void memcg_destroy_kmem_caches(struct mem_cgroup *memcg) 759 { 760 struct kmem_cache *s, *s2; 761 762 get_online_cpus(); 763 get_online_mems(); 764 765 mutex_lock(&slab_mutex); 766 list_for_each_entry_safe(s, s2, &memcg->kmem_caches, 767 memcg_params.kmem_caches_node) { 768 /* 769 * The cgroup is about to be freed and therefore has no charges 770 * left. Hence, all its caches must be empty by now. 771 */ 772 BUG_ON(shutdown_cache(s)); 773 } 774 mutex_unlock(&slab_mutex); 775 776 put_online_mems(); 777 put_online_cpus(); 778 } 779 780 static int shutdown_memcg_caches(struct kmem_cache *s) 781 { 782 struct memcg_cache_array *arr; 783 struct kmem_cache *c, *c2; 784 LIST_HEAD(busy); 785 int i; 786 787 BUG_ON(!is_root_cache(s)); 788 789 /* 790 * First, shutdown active caches, i.e. caches that belong to online 791 * memory cgroups. 792 */ 793 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 794 lockdep_is_held(&slab_mutex)); 795 for_each_memcg_cache_index(i) { 796 c = arr->entries[i]; 797 if (!c) 798 continue; 799 if (shutdown_cache(c)) 800 /* 801 * The cache still has objects. Move it to a temporary 802 * list so as not to try to destroy it for a second 803 * time while iterating over inactive caches below. 804 */ 805 list_move(&c->memcg_params.children_node, &busy); 806 else 807 /* 808 * The cache is empty and will be destroyed soon. Clear 809 * the pointer to it in the memcg_caches array so that 810 * it will never be accessed even if the root cache 811 * stays alive. 812 */ 813 arr->entries[i] = NULL; 814 } 815 816 /* 817 * Second, shutdown all caches left from memory cgroups that are now 818 * offline. 819 */ 820 list_for_each_entry_safe(c, c2, &s->memcg_params.children, 821 memcg_params.children_node) 822 shutdown_cache(c); 823 824 list_splice(&busy, &s->memcg_params.children); 825 826 /* 827 * A cache being destroyed must be empty. In particular, this means 828 * that all per memcg caches attached to it must be empty too. 829 */ 830 if (!list_empty(&s->memcg_params.children)) 831 return -EBUSY; 832 return 0; 833 } 834 835 static void flush_memcg_workqueue(struct kmem_cache *s) 836 { 837 mutex_lock(&slab_mutex); 838 s->memcg_params.dying = true; 839 mutex_unlock(&slab_mutex); 840 841 /* 842 * SLUB deactivates the kmem_caches through call_rcu_sched. Make 843 * sure all registered rcu callbacks have been invoked. 844 */ 845 if (IS_ENABLED(CONFIG_SLUB)) 846 rcu_barrier_sched(); 847 848 /* 849 * SLAB and SLUB create memcg kmem_caches through workqueue and SLUB 850 * deactivates the memcg kmem_caches through workqueue. Make sure all 851 * previous workitems on workqueue are processed. 852 */ 853 flush_workqueue(memcg_kmem_cache_wq); 854 } 855 #else 856 static inline int shutdown_memcg_caches(struct kmem_cache *s) 857 { 858 return 0; 859 } 860 861 static inline void flush_memcg_workqueue(struct kmem_cache *s) 862 { 863 } 864 #endif /* CONFIG_MEMCG_KMEM */ 865 866 void slab_kmem_cache_release(struct kmem_cache *s) 867 { 868 __kmem_cache_release(s); 869 destroy_memcg_params(s); 870 kfree_const(s->name); 871 kmem_cache_free(kmem_cache, s); 872 } 873 874 void kmem_cache_destroy(struct kmem_cache *s) 875 { 876 int err; 877 878 if (unlikely(!s)) 879 return; 880 881 flush_memcg_workqueue(s); 882 883 get_online_cpus(); 884 get_online_mems(); 885 886 mutex_lock(&slab_mutex); 887 888 s->refcount--; 889 if (s->refcount) 890 goto out_unlock; 891 892 err = shutdown_memcg_caches(s); 893 if (!err) 894 err = shutdown_cache(s); 895 896 if (err) { 897 pr_err("kmem_cache_destroy %s: Slab cache still has objects\n", 898 s->name); 899 dump_stack(); 900 } 901 out_unlock: 902 mutex_unlock(&slab_mutex); 903 904 put_online_mems(); 905 put_online_cpus(); 906 } 907 EXPORT_SYMBOL(kmem_cache_destroy); 908 909 /** 910 * kmem_cache_shrink - Shrink a cache. 911 * @cachep: The cache to shrink. 912 * 913 * Releases as many slabs as possible for a cache. 914 * To help debugging, a zero exit status indicates all slabs were released. 915 */ 916 int kmem_cache_shrink(struct kmem_cache *cachep) 917 { 918 int ret; 919 920 get_online_cpus(); 921 get_online_mems(); 922 kasan_cache_shrink(cachep); 923 ret = __kmem_cache_shrink(cachep); 924 put_online_mems(); 925 put_online_cpus(); 926 return ret; 927 } 928 EXPORT_SYMBOL(kmem_cache_shrink); 929 930 bool slab_is_available(void) 931 { 932 return slab_state >= UP; 933 } 934 935 #ifndef CONFIG_SLOB 936 /* Create a cache during boot when no slab services are available yet */ 937 void __init create_boot_cache(struct kmem_cache *s, const char *name, 938 unsigned int size, slab_flags_t flags, 939 unsigned int useroffset, unsigned int usersize) 940 { 941 int err; 942 943 s->name = name; 944 s->size = s->object_size = size; 945 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size); 946 s->useroffset = useroffset; 947 s->usersize = usersize; 948 949 slab_init_memcg_params(s); 950 951 err = __kmem_cache_create(s, flags); 952 953 if (err) 954 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n", 955 name, size, err); 956 957 s->refcount = -1; /* Exempt from merging for now */ 958 } 959 960 struct kmem_cache *__init create_kmalloc_cache(const char *name, 961 unsigned int size, slab_flags_t flags, 962 unsigned int useroffset, unsigned int usersize) 963 { 964 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 965 966 if (!s) 967 panic("Out of memory when creating slab %s\n", name); 968 969 create_boot_cache(s, name, size, flags, useroffset, usersize); 970 list_add(&s->list, &slab_caches); 971 memcg_link_cache(s); 972 s->refcount = 1; 973 return s; 974 } 975 976 struct kmem_cache * 977 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init; 978 EXPORT_SYMBOL(kmalloc_caches); 979 980 /* 981 * Conversion table for small slabs sizes / 8 to the index in the 982 * kmalloc array. This is necessary for slabs < 192 since we have non power 983 * of two cache sizes there. The size of larger slabs can be determined using 984 * fls. 985 */ 986 static u8 size_index[24] __ro_after_init = { 987 3, /* 8 */ 988 4, /* 16 */ 989 5, /* 24 */ 990 5, /* 32 */ 991 6, /* 40 */ 992 6, /* 48 */ 993 6, /* 56 */ 994 6, /* 64 */ 995 1, /* 72 */ 996 1, /* 80 */ 997 1, /* 88 */ 998 1, /* 96 */ 999 7, /* 104 */ 1000 7, /* 112 */ 1001 7, /* 120 */ 1002 7, /* 128 */ 1003 2, /* 136 */ 1004 2, /* 144 */ 1005 2, /* 152 */ 1006 2, /* 160 */ 1007 2, /* 168 */ 1008 2, /* 176 */ 1009 2, /* 184 */ 1010 2 /* 192 */ 1011 }; 1012 1013 static inline unsigned int size_index_elem(unsigned int bytes) 1014 { 1015 return (bytes - 1) / 8; 1016 } 1017 1018 /* 1019 * Find the kmem_cache structure that serves a given size of 1020 * allocation 1021 */ 1022 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) 1023 { 1024 unsigned int index; 1025 1026 if (size <= 192) { 1027 if (!size) 1028 return ZERO_SIZE_PTR; 1029 1030 index = size_index[size_index_elem(size)]; 1031 } else { 1032 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 1033 WARN_ON(1); 1034 return NULL; 1035 } 1036 index = fls(size - 1); 1037 } 1038 1039 return kmalloc_caches[kmalloc_type(flags)][index]; 1040 } 1041 1042 /* 1043 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. 1044 * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is 1045 * kmalloc-67108864. 1046 */ 1047 const struct kmalloc_info_struct kmalloc_info[] __initconst = { 1048 {NULL, 0}, {"kmalloc-96", 96}, 1049 {"kmalloc-192", 192}, {"kmalloc-8", 8}, 1050 {"kmalloc-16", 16}, {"kmalloc-32", 32}, 1051 {"kmalloc-64", 64}, {"kmalloc-128", 128}, 1052 {"kmalloc-256", 256}, {"kmalloc-512", 512}, 1053 {"kmalloc-1k", 1024}, {"kmalloc-2k", 2048}, 1054 {"kmalloc-4k", 4096}, {"kmalloc-8k", 8192}, 1055 {"kmalloc-16k", 16384}, {"kmalloc-32k", 32768}, 1056 {"kmalloc-64k", 65536}, {"kmalloc-128k", 131072}, 1057 {"kmalloc-256k", 262144}, {"kmalloc-512k", 524288}, 1058 {"kmalloc-1M", 1048576}, {"kmalloc-2M", 2097152}, 1059 {"kmalloc-4M", 4194304}, {"kmalloc-8M", 8388608}, 1060 {"kmalloc-16M", 16777216}, {"kmalloc-32M", 33554432}, 1061 {"kmalloc-64M", 67108864} 1062 }; 1063 1064 /* 1065 * Patch up the size_index table if we have strange large alignment 1066 * requirements for the kmalloc array. This is only the case for 1067 * MIPS it seems. The standard arches will not generate any code here. 1068 * 1069 * Largest permitted alignment is 256 bytes due to the way we 1070 * handle the index determination for the smaller caches. 1071 * 1072 * Make sure that nothing crazy happens if someone starts tinkering 1073 * around with ARCH_KMALLOC_MINALIGN 1074 */ 1075 void __init setup_kmalloc_cache_index_table(void) 1076 { 1077 unsigned int i; 1078 1079 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 1080 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1))); 1081 1082 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) { 1083 unsigned int elem = size_index_elem(i); 1084 1085 if (elem >= ARRAY_SIZE(size_index)) 1086 break; 1087 size_index[elem] = KMALLOC_SHIFT_LOW; 1088 } 1089 1090 if (KMALLOC_MIN_SIZE >= 64) { 1091 /* 1092 * The 96 byte size cache is not used if the alignment 1093 * is 64 byte. 1094 */ 1095 for (i = 64 + 8; i <= 96; i += 8) 1096 size_index[size_index_elem(i)] = 7; 1097 1098 } 1099 1100 if (KMALLOC_MIN_SIZE >= 128) { 1101 /* 1102 * The 192 byte sized cache is not used if the alignment 1103 * is 128 byte. Redirect kmalloc to use the 256 byte cache 1104 * instead. 1105 */ 1106 for (i = 128 + 8; i <= 192; i += 8) 1107 size_index[size_index_elem(i)] = 8; 1108 } 1109 } 1110 1111 static const char * 1112 kmalloc_cache_name(const char *prefix, unsigned int size) 1113 { 1114 1115 static const char units[3] = "\0kM"; 1116 int idx = 0; 1117 1118 while (size >= 1024 && (size % 1024 == 0)) { 1119 size /= 1024; 1120 idx++; 1121 } 1122 1123 return kasprintf(GFP_NOWAIT, "%s-%u%c", prefix, size, units[idx]); 1124 } 1125 1126 static void __init 1127 new_kmalloc_cache(int idx, int type, slab_flags_t flags) 1128 { 1129 const char *name; 1130 1131 if (type == KMALLOC_RECLAIM) { 1132 flags |= SLAB_RECLAIM_ACCOUNT; 1133 name = kmalloc_cache_name("kmalloc-rcl", 1134 kmalloc_info[idx].size); 1135 BUG_ON(!name); 1136 } else { 1137 name = kmalloc_info[idx].name; 1138 } 1139 1140 kmalloc_caches[type][idx] = create_kmalloc_cache(name, 1141 kmalloc_info[idx].size, flags, 0, 1142 kmalloc_info[idx].size); 1143 } 1144 1145 /* 1146 * Create the kmalloc array. Some of the regular kmalloc arrays 1147 * may already have been created because they were needed to 1148 * enable allocations for slab creation. 1149 */ 1150 void __init create_kmalloc_caches(slab_flags_t flags) 1151 { 1152 int i, type; 1153 1154 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) { 1155 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 1156 if (!kmalloc_caches[type][i]) 1157 new_kmalloc_cache(i, type, flags); 1158 1159 /* 1160 * Caches that are not of the two-to-the-power-of size. 1161 * These have to be created immediately after the 1162 * earlier power of two caches 1163 */ 1164 if (KMALLOC_MIN_SIZE <= 32 && i == 6 && 1165 !kmalloc_caches[type][1]) 1166 new_kmalloc_cache(1, type, flags); 1167 if (KMALLOC_MIN_SIZE <= 64 && i == 7 && 1168 !kmalloc_caches[type][2]) 1169 new_kmalloc_cache(2, type, flags); 1170 } 1171 } 1172 1173 /* Kmalloc array is now usable */ 1174 slab_state = UP; 1175 1176 #ifdef CONFIG_ZONE_DMA 1177 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) { 1178 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i]; 1179 1180 if (s) { 1181 unsigned int size = kmalloc_size(i); 1182 const char *n = kmalloc_cache_name("dma-kmalloc", size); 1183 1184 BUG_ON(!n); 1185 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache( 1186 n, size, SLAB_CACHE_DMA | flags, 0, 0); 1187 } 1188 } 1189 #endif 1190 } 1191 #endif /* !CONFIG_SLOB */ 1192 1193 /* 1194 * To avoid unnecessary overhead, we pass through large allocation requests 1195 * directly to the page allocator. We use __GFP_COMP, because we will need to 1196 * know the allocation order to free the pages properly in kfree. 1197 */ 1198 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) 1199 { 1200 void *ret; 1201 struct page *page; 1202 1203 flags |= __GFP_COMP; 1204 page = alloc_pages(flags, order); 1205 ret = page ? page_address(page) : NULL; 1206 kmemleak_alloc(ret, size, 1, flags); 1207 kasan_kmalloc_large(ret, size, flags); 1208 return ret; 1209 } 1210 EXPORT_SYMBOL(kmalloc_order); 1211 1212 #ifdef CONFIG_TRACING 1213 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 1214 { 1215 void *ret = kmalloc_order(size, flags, order); 1216 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 1217 return ret; 1218 } 1219 EXPORT_SYMBOL(kmalloc_order_trace); 1220 #endif 1221 1222 #ifdef CONFIG_SLAB_FREELIST_RANDOM 1223 /* Randomize a generic freelist */ 1224 static void freelist_randomize(struct rnd_state *state, unsigned int *list, 1225 unsigned int count) 1226 { 1227 unsigned int rand; 1228 unsigned int i; 1229 1230 for (i = 0; i < count; i++) 1231 list[i] = i; 1232 1233 /* Fisher-Yates shuffle */ 1234 for (i = count - 1; i > 0; i--) { 1235 rand = prandom_u32_state(state); 1236 rand %= (i + 1); 1237 swap(list[i], list[rand]); 1238 } 1239 } 1240 1241 /* Create a random sequence per cache */ 1242 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 1243 gfp_t gfp) 1244 { 1245 struct rnd_state state; 1246 1247 if (count < 2 || cachep->random_seq) 1248 return 0; 1249 1250 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp); 1251 if (!cachep->random_seq) 1252 return -ENOMEM; 1253 1254 /* Get best entropy at this stage of boot */ 1255 prandom_seed_state(&state, get_random_long()); 1256 1257 freelist_randomize(&state, cachep->random_seq, count); 1258 return 0; 1259 } 1260 1261 /* Destroy the per-cache random freelist sequence */ 1262 void cache_random_seq_destroy(struct kmem_cache *cachep) 1263 { 1264 kfree(cachep->random_seq); 1265 cachep->random_seq = NULL; 1266 } 1267 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 1268 1269 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) 1270 #ifdef CONFIG_SLAB 1271 #define SLABINFO_RIGHTS (0600) 1272 #else 1273 #define SLABINFO_RIGHTS (0400) 1274 #endif 1275 1276 static void print_slabinfo_header(struct seq_file *m) 1277 { 1278 /* 1279 * Output format version, so at least we can change it 1280 * without _too_ many complaints. 1281 */ 1282 #ifdef CONFIG_DEBUG_SLAB 1283 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 1284 #else 1285 seq_puts(m, "slabinfo - version: 2.1\n"); 1286 #endif 1287 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>"); 1288 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 1289 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 1290 #ifdef CONFIG_DEBUG_SLAB 1291 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 1292 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 1293 #endif 1294 seq_putc(m, '\n'); 1295 } 1296 1297 void *slab_start(struct seq_file *m, loff_t *pos) 1298 { 1299 mutex_lock(&slab_mutex); 1300 return seq_list_start(&slab_root_caches, *pos); 1301 } 1302 1303 void *slab_next(struct seq_file *m, void *p, loff_t *pos) 1304 { 1305 return seq_list_next(p, &slab_root_caches, pos); 1306 } 1307 1308 void slab_stop(struct seq_file *m, void *p) 1309 { 1310 mutex_unlock(&slab_mutex); 1311 } 1312 1313 static void 1314 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info) 1315 { 1316 struct kmem_cache *c; 1317 struct slabinfo sinfo; 1318 1319 if (!is_root_cache(s)) 1320 return; 1321 1322 for_each_memcg_cache(c, s) { 1323 memset(&sinfo, 0, sizeof(sinfo)); 1324 get_slabinfo(c, &sinfo); 1325 1326 info->active_slabs += sinfo.active_slabs; 1327 info->num_slabs += sinfo.num_slabs; 1328 info->shared_avail += sinfo.shared_avail; 1329 info->active_objs += sinfo.active_objs; 1330 info->num_objs += sinfo.num_objs; 1331 } 1332 } 1333 1334 static void cache_show(struct kmem_cache *s, struct seq_file *m) 1335 { 1336 struct slabinfo sinfo; 1337 1338 memset(&sinfo, 0, sizeof(sinfo)); 1339 get_slabinfo(s, &sinfo); 1340 1341 memcg_accumulate_slabinfo(s, &sinfo); 1342 1343 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 1344 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, 1345 sinfo.objects_per_slab, (1 << sinfo.cache_order)); 1346 1347 seq_printf(m, " : tunables %4u %4u %4u", 1348 sinfo.limit, sinfo.batchcount, sinfo.shared); 1349 seq_printf(m, " : slabdata %6lu %6lu %6lu", 1350 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); 1351 slabinfo_show_stats(m, s); 1352 seq_putc(m, '\n'); 1353 } 1354 1355 static int slab_show(struct seq_file *m, void *p) 1356 { 1357 struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node); 1358 1359 if (p == slab_root_caches.next) 1360 print_slabinfo_header(m); 1361 cache_show(s, m); 1362 return 0; 1363 } 1364 1365 void dump_unreclaimable_slab(void) 1366 { 1367 struct kmem_cache *s, *s2; 1368 struct slabinfo sinfo; 1369 1370 /* 1371 * Here acquiring slab_mutex is risky since we don't prefer to get 1372 * sleep in oom path. But, without mutex hold, it may introduce a 1373 * risk of crash. 1374 * Use mutex_trylock to protect the list traverse, dump nothing 1375 * without acquiring the mutex. 1376 */ 1377 if (!mutex_trylock(&slab_mutex)) { 1378 pr_warn("excessive unreclaimable slab but cannot dump stats\n"); 1379 return; 1380 } 1381 1382 pr_info("Unreclaimable slab info:\n"); 1383 pr_info("Name Used Total\n"); 1384 1385 list_for_each_entry_safe(s, s2, &slab_caches, list) { 1386 if (!is_root_cache(s) || (s->flags & SLAB_RECLAIM_ACCOUNT)) 1387 continue; 1388 1389 get_slabinfo(s, &sinfo); 1390 1391 if (sinfo.num_objs > 0) 1392 pr_info("%-17s %10luKB %10luKB\n", cache_name(s), 1393 (sinfo.active_objs * s->size) / 1024, 1394 (sinfo.num_objs * s->size) / 1024); 1395 } 1396 mutex_unlock(&slab_mutex); 1397 } 1398 1399 #if defined(CONFIG_MEMCG) 1400 void *memcg_slab_start(struct seq_file *m, loff_t *pos) 1401 { 1402 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1403 1404 mutex_lock(&slab_mutex); 1405 return seq_list_start(&memcg->kmem_caches, *pos); 1406 } 1407 1408 void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos) 1409 { 1410 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1411 1412 return seq_list_next(p, &memcg->kmem_caches, pos); 1413 } 1414 1415 void memcg_slab_stop(struct seq_file *m, void *p) 1416 { 1417 mutex_unlock(&slab_mutex); 1418 } 1419 1420 int memcg_slab_show(struct seq_file *m, void *p) 1421 { 1422 struct kmem_cache *s = list_entry(p, struct kmem_cache, 1423 memcg_params.kmem_caches_node); 1424 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1425 1426 if (p == memcg->kmem_caches.next) 1427 print_slabinfo_header(m); 1428 cache_show(s, m); 1429 return 0; 1430 } 1431 #endif 1432 1433 /* 1434 * slabinfo_op - iterator that generates /proc/slabinfo 1435 * 1436 * Output layout: 1437 * cache-name 1438 * num-active-objs 1439 * total-objs 1440 * object size 1441 * num-active-slabs 1442 * total-slabs 1443 * num-pages-per-slab 1444 * + further values on SMP and with statistics enabled 1445 */ 1446 static const struct seq_operations slabinfo_op = { 1447 .start = slab_start, 1448 .next = slab_next, 1449 .stop = slab_stop, 1450 .show = slab_show, 1451 }; 1452 1453 static int slabinfo_open(struct inode *inode, struct file *file) 1454 { 1455 return seq_open(file, &slabinfo_op); 1456 } 1457 1458 static const struct file_operations proc_slabinfo_operations = { 1459 .open = slabinfo_open, 1460 .read = seq_read, 1461 .write = slabinfo_write, 1462 .llseek = seq_lseek, 1463 .release = seq_release, 1464 }; 1465 1466 static int __init slab_proc_init(void) 1467 { 1468 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, 1469 &proc_slabinfo_operations); 1470 return 0; 1471 } 1472 module_init(slab_proc_init); 1473 #endif /* CONFIG_SLAB || CONFIG_SLUB_DEBUG */ 1474 1475 static __always_inline void *__do_krealloc(const void *p, size_t new_size, 1476 gfp_t flags) 1477 { 1478 void *ret; 1479 size_t ks = 0; 1480 1481 if (p) 1482 ks = ksize(p); 1483 1484 if (ks >= new_size) { 1485 kasan_krealloc((void *)p, new_size, flags); 1486 return (void *)p; 1487 } 1488 1489 ret = kmalloc_track_caller(new_size, flags); 1490 if (ret && p) 1491 memcpy(ret, p, ks); 1492 1493 return ret; 1494 } 1495 1496 /** 1497 * __krealloc - like krealloc() but don't free @p. 1498 * @p: object to reallocate memory for. 1499 * @new_size: how many bytes of memory are required. 1500 * @flags: the type of memory to allocate. 1501 * 1502 * This function is like krealloc() except it never frees the originally 1503 * allocated buffer. Use this if you don't want to free the buffer immediately 1504 * like, for example, with RCU. 1505 */ 1506 void *__krealloc(const void *p, size_t new_size, gfp_t flags) 1507 { 1508 if (unlikely(!new_size)) 1509 return ZERO_SIZE_PTR; 1510 1511 return __do_krealloc(p, new_size, flags); 1512 1513 } 1514 EXPORT_SYMBOL(__krealloc); 1515 1516 /** 1517 * krealloc - reallocate memory. The contents will remain unchanged. 1518 * @p: object to reallocate memory for. 1519 * @new_size: how many bytes of memory are required. 1520 * @flags: the type of memory to allocate. 1521 * 1522 * The contents of the object pointed to are preserved up to the 1523 * lesser of the new and old sizes. If @p is %NULL, krealloc() 1524 * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a 1525 * %NULL pointer, the object pointed to is freed. 1526 */ 1527 void *krealloc(const void *p, size_t new_size, gfp_t flags) 1528 { 1529 void *ret; 1530 1531 if (unlikely(!new_size)) { 1532 kfree(p); 1533 return ZERO_SIZE_PTR; 1534 } 1535 1536 ret = __do_krealloc(p, new_size, flags); 1537 if (ret && p != ret) 1538 kfree(p); 1539 1540 return ret; 1541 } 1542 EXPORT_SYMBOL(krealloc); 1543 1544 /** 1545 * kzfree - like kfree but zero memory 1546 * @p: object to free memory of 1547 * 1548 * The memory of the object @p points to is zeroed before freed. 1549 * If @p is %NULL, kzfree() does nothing. 1550 * 1551 * Note: this function zeroes the whole allocated buffer which can be a good 1552 * deal bigger than the requested buffer size passed to kmalloc(). So be 1553 * careful when using this function in performance sensitive code. 1554 */ 1555 void kzfree(const void *p) 1556 { 1557 size_t ks; 1558 void *mem = (void *)p; 1559 1560 if (unlikely(ZERO_OR_NULL_PTR(mem))) 1561 return; 1562 ks = ksize(mem); 1563 memset(mem, 0, ks); 1564 kfree(mem); 1565 } 1566 EXPORT_SYMBOL(kzfree); 1567 1568 /* Tracepoints definitions. */ 1569 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 1570 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 1571 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); 1572 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); 1573 EXPORT_TRACEPOINT_SYMBOL(kfree); 1574 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1575 1576 int should_failslab(struct kmem_cache *s, gfp_t gfpflags) 1577 { 1578 if (__should_failslab(s, gfpflags)) 1579 return -ENOMEM; 1580 return 0; 1581 } 1582 ALLOW_ERROR_INJECTION(should_failslab, ERRNO); 1583