1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Slab allocator functions that are independent of the allocator strategy 4 * 5 * (C) 2012 Christoph Lameter <cl@linux.com> 6 */ 7 #include <linux/slab.h> 8 9 #include <linux/mm.h> 10 #include <linux/poison.h> 11 #include <linux/interrupt.h> 12 #include <linux/memory.h> 13 #include <linux/cache.h> 14 #include <linux/compiler.h> 15 #include <linux/module.h> 16 #include <linux/cpu.h> 17 #include <linux/uaccess.h> 18 #include <linux/seq_file.h> 19 #include <linux/proc_fs.h> 20 #include <asm/cacheflush.h> 21 #include <asm/tlbflush.h> 22 #include <asm/page.h> 23 #include <linux/memcontrol.h> 24 25 #define CREATE_TRACE_POINTS 26 #include <trace/events/kmem.h> 27 28 #include "slab.h" 29 30 enum slab_state slab_state; 31 LIST_HEAD(slab_caches); 32 DEFINE_MUTEX(slab_mutex); 33 struct kmem_cache *kmem_cache; 34 35 #ifdef CONFIG_HARDENED_USERCOPY 36 bool usercopy_fallback __ro_after_init = 37 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK); 38 module_param(usercopy_fallback, bool, 0400); 39 MODULE_PARM_DESC(usercopy_fallback, 40 "WARN instead of reject usercopy whitelist violations"); 41 #endif 42 43 static LIST_HEAD(slab_caches_to_rcu_destroy); 44 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work); 45 static DECLARE_WORK(slab_caches_to_rcu_destroy_work, 46 slab_caches_to_rcu_destroy_workfn); 47 48 /* 49 * Set of flags that will prevent slab merging 50 */ 51 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 52 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ 53 SLAB_FAILSLAB | SLAB_KASAN) 54 55 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ 56 SLAB_ACCOUNT) 57 58 /* 59 * Merge control. If this is set then no merging of slab caches will occur. 60 */ 61 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT); 62 63 static int __init setup_slab_nomerge(char *str) 64 { 65 slab_nomerge = true; 66 return 1; 67 } 68 69 #ifdef CONFIG_SLUB 70 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0); 71 #endif 72 73 __setup("slab_nomerge", setup_slab_nomerge); 74 75 /* 76 * Determine the size of a slab object 77 */ 78 unsigned int kmem_cache_size(struct kmem_cache *s) 79 { 80 return s->object_size; 81 } 82 EXPORT_SYMBOL(kmem_cache_size); 83 84 #ifdef CONFIG_DEBUG_VM 85 static int kmem_cache_sanity_check(const char *name, unsigned int size) 86 { 87 if (!name || in_interrupt() || size < sizeof(void *) || 88 size > KMALLOC_MAX_SIZE) { 89 pr_err("kmem_cache_create(%s) integrity check failed\n", name); 90 return -EINVAL; 91 } 92 93 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 94 return 0; 95 } 96 #else 97 static inline int kmem_cache_sanity_check(const char *name, unsigned int size) 98 { 99 return 0; 100 } 101 #endif 102 103 void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 104 { 105 size_t i; 106 107 for (i = 0; i < nr; i++) { 108 if (s) 109 kmem_cache_free(s, p[i]); 110 else 111 kfree(p[i]); 112 } 113 } 114 115 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 116 void **p) 117 { 118 size_t i; 119 120 for (i = 0; i < nr; i++) { 121 void *x = p[i] = kmem_cache_alloc(s, flags); 122 if (!x) { 123 __kmem_cache_free_bulk(s, i, p); 124 return 0; 125 } 126 } 127 return i; 128 } 129 130 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 131 132 LIST_HEAD(slab_root_caches); 133 134 void slab_init_memcg_params(struct kmem_cache *s) 135 { 136 s->memcg_params.root_cache = NULL; 137 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); 138 INIT_LIST_HEAD(&s->memcg_params.children); 139 s->memcg_params.dying = false; 140 } 141 142 static int init_memcg_params(struct kmem_cache *s, 143 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 144 { 145 struct memcg_cache_array *arr; 146 147 if (root_cache) { 148 s->memcg_params.root_cache = root_cache; 149 s->memcg_params.memcg = memcg; 150 INIT_LIST_HEAD(&s->memcg_params.children_node); 151 INIT_LIST_HEAD(&s->memcg_params.kmem_caches_node); 152 return 0; 153 } 154 155 slab_init_memcg_params(s); 156 157 if (!memcg_nr_cache_ids) 158 return 0; 159 160 arr = kvzalloc(sizeof(struct memcg_cache_array) + 161 memcg_nr_cache_ids * sizeof(void *), 162 GFP_KERNEL); 163 if (!arr) 164 return -ENOMEM; 165 166 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr); 167 return 0; 168 } 169 170 static void destroy_memcg_params(struct kmem_cache *s) 171 { 172 if (is_root_cache(s)) 173 kvfree(rcu_access_pointer(s->memcg_params.memcg_caches)); 174 } 175 176 static void free_memcg_params(struct rcu_head *rcu) 177 { 178 struct memcg_cache_array *old; 179 180 old = container_of(rcu, struct memcg_cache_array, rcu); 181 kvfree(old); 182 } 183 184 static int update_memcg_params(struct kmem_cache *s, int new_array_size) 185 { 186 struct memcg_cache_array *old, *new; 187 188 new = kvzalloc(sizeof(struct memcg_cache_array) + 189 new_array_size * sizeof(void *), GFP_KERNEL); 190 if (!new) 191 return -ENOMEM; 192 193 old = rcu_dereference_protected(s->memcg_params.memcg_caches, 194 lockdep_is_held(&slab_mutex)); 195 if (old) 196 memcpy(new->entries, old->entries, 197 memcg_nr_cache_ids * sizeof(void *)); 198 199 rcu_assign_pointer(s->memcg_params.memcg_caches, new); 200 if (old) 201 call_rcu(&old->rcu, free_memcg_params); 202 return 0; 203 } 204 205 int memcg_update_all_caches(int num_memcgs) 206 { 207 struct kmem_cache *s; 208 int ret = 0; 209 210 mutex_lock(&slab_mutex); 211 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 212 ret = update_memcg_params(s, num_memcgs); 213 /* 214 * Instead of freeing the memory, we'll just leave the caches 215 * up to this point in an updated state. 216 */ 217 if (ret) 218 break; 219 } 220 mutex_unlock(&slab_mutex); 221 return ret; 222 } 223 224 void memcg_link_cache(struct kmem_cache *s) 225 { 226 if (is_root_cache(s)) { 227 list_add(&s->root_caches_node, &slab_root_caches); 228 } else { 229 list_add(&s->memcg_params.children_node, 230 &s->memcg_params.root_cache->memcg_params.children); 231 list_add(&s->memcg_params.kmem_caches_node, 232 &s->memcg_params.memcg->kmem_caches); 233 } 234 } 235 236 static void memcg_unlink_cache(struct kmem_cache *s) 237 { 238 if (is_root_cache(s)) { 239 list_del(&s->root_caches_node); 240 } else { 241 list_del(&s->memcg_params.children_node); 242 list_del(&s->memcg_params.kmem_caches_node); 243 } 244 } 245 #else 246 static inline int init_memcg_params(struct kmem_cache *s, 247 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 248 { 249 return 0; 250 } 251 252 static inline void destroy_memcg_params(struct kmem_cache *s) 253 { 254 } 255 256 static inline void memcg_unlink_cache(struct kmem_cache *s) 257 { 258 } 259 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */ 260 261 /* 262 * Figure out what the alignment of the objects will be given a set of 263 * flags, a user specified alignment and the size of the objects. 264 */ 265 static unsigned int calculate_alignment(slab_flags_t flags, 266 unsigned int align, unsigned int size) 267 { 268 /* 269 * If the user wants hardware cache aligned objects then follow that 270 * suggestion if the object is sufficiently large. 271 * 272 * The hardware cache alignment cannot override the specified 273 * alignment though. If that is greater then use it. 274 */ 275 if (flags & SLAB_HWCACHE_ALIGN) { 276 unsigned int ralign; 277 278 ralign = cache_line_size(); 279 while (size <= ralign / 2) 280 ralign /= 2; 281 align = max(align, ralign); 282 } 283 284 if (align < ARCH_SLAB_MINALIGN) 285 align = ARCH_SLAB_MINALIGN; 286 287 return ALIGN(align, sizeof(void *)); 288 } 289 290 /* 291 * Find a mergeable slab cache 292 */ 293 int slab_unmergeable(struct kmem_cache *s) 294 { 295 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) 296 return 1; 297 298 if (!is_root_cache(s)) 299 return 1; 300 301 if (s->ctor) 302 return 1; 303 304 if (s->usersize) 305 return 1; 306 307 /* 308 * We may have set a slab to be unmergeable during bootstrap. 309 */ 310 if (s->refcount < 0) 311 return 1; 312 313 return 0; 314 } 315 316 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, 317 slab_flags_t flags, const char *name, void (*ctor)(void *)) 318 { 319 struct kmem_cache *s; 320 321 if (slab_nomerge) 322 return NULL; 323 324 if (ctor) 325 return NULL; 326 327 size = ALIGN(size, sizeof(void *)); 328 align = calculate_alignment(flags, align, size); 329 size = ALIGN(size, align); 330 flags = kmem_cache_flags(size, flags, name, NULL); 331 332 if (flags & SLAB_NEVER_MERGE) 333 return NULL; 334 335 list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) { 336 if (slab_unmergeable(s)) 337 continue; 338 339 if (size > s->size) 340 continue; 341 342 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) 343 continue; 344 /* 345 * Check if alignment is compatible. 346 * Courtesy of Adrian Drzewiecki 347 */ 348 if ((s->size & ~(align - 1)) != s->size) 349 continue; 350 351 if (s->size - size >= sizeof(void *)) 352 continue; 353 354 if (IS_ENABLED(CONFIG_SLAB) && align && 355 (align > s->align || s->align % align)) 356 continue; 357 358 return s; 359 } 360 return NULL; 361 } 362 363 static struct kmem_cache *create_cache(const char *name, 364 unsigned int object_size, unsigned int align, 365 slab_flags_t flags, unsigned int useroffset, 366 unsigned int usersize, void (*ctor)(void *), 367 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 368 { 369 struct kmem_cache *s; 370 int err; 371 372 if (WARN_ON(useroffset + usersize > object_size)) 373 useroffset = usersize = 0; 374 375 err = -ENOMEM; 376 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 377 if (!s) 378 goto out; 379 380 s->name = name; 381 s->size = s->object_size = object_size; 382 s->align = align; 383 s->ctor = ctor; 384 s->useroffset = useroffset; 385 s->usersize = usersize; 386 387 err = init_memcg_params(s, memcg, root_cache); 388 if (err) 389 goto out_free_cache; 390 391 err = __kmem_cache_create(s, flags); 392 if (err) 393 goto out_free_cache; 394 395 s->refcount = 1; 396 list_add(&s->list, &slab_caches); 397 memcg_link_cache(s); 398 out: 399 if (err) 400 return ERR_PTR(err); 401 return s; 402 403 out_free_cache: 404 destroy_memcg_params(s); 405 kmem_cache_free(kmem_cache, s); 406 goto out; 407 } 408 409 /* 410 * kmem_cache_create_usercopy - Create a cache. 411 * @name: A string which is used in /proc/slabinfo to identify this cache. 412 * @size: The size of objects to be created in this cache. 413 * @align: The required alignment for the objects. 414 * @flags: SLAB flags 415 * @useroffset: Usercopy region offset 416 * @usersize: Usercopy region size 417 * @ctor: A constructor for the objects. 418 * 419 * Returns a ptr to the cache on success, NULL on failure. 420 * Cannot be called within a interrupt, but can be interrupted. 421 * The @ctor is run when new pages are allocated by the cache. 422 * 423 * The flags are 424 * 425 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 426 * to catch references to uninitialised memory. 427 * 428 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 429 * for buffer overruns. 430 * 431 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 432 * cacheline. This can be beneficial if you're counting cycles as closely 433 * as davem. 434 */ 435 struct kmem_cache * 436 kmem_cache_create_usercopy(const char *name, 437 unsigned int size, unsigned int align, 438 slab_flags_t flags, 439 unsigned int useroffset, unsigned int usersize, 440 void (*ctor)(void *)) 441 { 442 struct kmem_cache *s = NULL; 443 const char *cache_name; 444 int err; 445 446 get_online_cpus(); 447 get_online_mems(); 448 memcg_get_cache_ids(); 449 450 mutex_lock(&slab_mutex); 451 452 err = kmem_cache_sanity_check(name, size); 453 if (err) { 454 goto out_unlock; 455 } 456 457 /* Refuse requests with allocator specific flags */ 458 if (flags & ~SLAB_FLAGS_PERMITTED) { 459 err = -EINVAL; 460 goto out_unlock; 461 } 462 463 /* 464 * Some allocators will constraint the set of valid flags to a subset 465 * of all flags. We expect them to define CACHE_CREATE_MASK in this 466 * case, and we'll just provide them with a sanitized version of the 467 * passed flags. 468 */ 469 flags &= CACHE_CREATE_MASK; 470 471 /* Fail closed on bad usersize of useroffset values. */ 472 if (WARN_ON(!usersize && useroffset) || 473 WARN_ON(size < usersize || size - usersize < useroffset)) 474 usersize = useroffset = 0; 475 476 if (!usersize) 477 s = __kmem_cache_alias(name, size, align, flags, ctor); 478 if (s) 479 goto out_unlock; 480 481 cache_name = kstrdup_const(name, GFP_KERNEL); 482 if (!cache_name) { 483 err = -ENOMEM; 484 goto out_unlock; 485 } 486 487 s = create_cache(cache_name, size, 488 calculate_alignment(flags, align, size), 489 flags, useroffset, usersize, ctor, NULL, NULL); 490 if (IS_ERR(s)) { 491 err = PTR_ERR(s); 492 kfree_const(cache_name); 493 } 494 495 out_unlock: 496 mutex_unlock(&slab_mutex); 497 498 memcg_put_cache_ids(); 499 put_online_mems(); 500 put_online_cpus(); 501 502 if (err) { 503 if (flags & SLAB_PANIC) 504 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", 505 name, err); 506 else { 507 pr_warn("kmem_cache_create(%s) failed with error %d\n", 508 name, err); 509 dump_stack(); 510 } 511 return NULL; 512 } 513 return s; 514 } 515 EXPORT_SYMBOL(kmem_cache_create_usercopy); 516 517 struct kmem_cache * 518 kmem_cache_create(const char *name, unsigned int size, unsigned int align, 519 slab_flags_t flags, void (*ctor)(void *)) 520 { 521 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0, 522 ctor); 523 } 524 EXPORT_SYMBOL(kmem_cache_create); 525 526 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) 527 { 528 LIST_HEAD(to_destroy); 529 struct kmem_cache *s, *s2; 530 531 /* 532 * On destruction, SLAB_TYPESAFE_BY_RCU kmem_caches are put on the 533 * @slab_caches_to_rcu_destroy list. The slab pages are freed 534 * through RCU and and the associated kmem_cache are dereferenced 535 * while freeing the pages, so the kmem_caches should be freed only 536 * after the pending RCU operations are finished. As rcu_barrier() 537 * is a pretty slow operation, we batch all pending destructions 538 * asynchronously. 539 */ 540 mutex_lock(&slab_mutex); 541 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy); 542 mutex_unlock(&slab_mutex); 543 544 if (list_empty(&to_destroy)) 545 return; 546 547 rcu_barrier(); 548 549 list_for_each_entry_safe(s, s2, &to_destroy, list) { 550 #ifdef SLAB_SUPPORTS_SYSFS 551 sysfs_slab_release(s); 552 #else 553 slab_kmem_cache_release(s); 554 #endif 555 } 556 } 557 558 static int shutdown_cache(struct kmem_cache *s) 559 { 560 /* free asan quarantined objects */ 561 kasan_cache_shutdown(s); 562 563 if (__kmem_cache_shutdown(s) != 0) 564 return -EBUSY; 565 566 memcg_unlink_cache(s); 567 list_del(&s->list); 568 569 if (s->flags & SLAB_TYPESAFE_BY_RCU) { 570 list_add_tail(&s->list, &slab_caches_to_rcu_destroy); 571 schedule_work(&slab_caches_to_rcu_destroy_work); 572 } else { 573 #ifdef SLAB_SUPPORTS_SYSFS 574 sysfs_slab_release(s); 575 #else 576 slab_kmem_cache_release(s); 577 #endif 578 } 579 580 return 0; 581 } 582 583 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 584 /* 585 * memcg_create_kmem_cache - Create a cache for a memory cgroup. 586 * @memcg: The memory cgroup the new cache is for. 587 * @root_cache: The parent of the new cache. 588 * 589 * This function attempts to create a kmem cache that will serve allocation 590 * requests going from @memcg to @root_cache. The new cache inherits properties 591 * from its parent. 592 */ 593 void memcg_create_kmem_cache(struct mem_cgroup *memcg, 594 struct kmem_cache *root_cache) 595 { 596 static char memcg_name_buf[NAME_MAX + 1]; /* protected by slab_mutex */ 597 struct cgroup_subsys_state *css = &memcg->css; 598 struct memcg_cache_array *arr; 599 struct kmem_cache *s = NULL; 600 char *cache_name; 601 int idx; 602 603 get_online_cpus(); 604 get_online_mems(); 605 606 mutex_lock(&slab_mutex); 607 608 /* 609 * The memory cgroup could have been offlined while the cache 610 * creation work was pending. 611 */ 612 if (memcg->kmem_state != KMEM_ONLINE || root_cache->memcg_params.dying) 613 goto out_unlock; 614 615 idx = memcg_cache_id(memcg); 616 arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches, 617 lockdep_is_held(&slab_mutex)); 618 619 /* 620 * Since per-memcg caches are created asynchronously on first 621 * allocation (see memcg_kmem_get_cache()), several threads can try to 622 * create the same cache, but only one of them may succeed. 623 */ 624 if (arr->entries[idx]) 625 goto out_unlock; 626 627 cgroup_name(css->cgroup, memcg_name_buf, sizeof(memcg_name_buf)); 628 cache_name = kasprintf(GFP_KERNEL, "%s(%llu:%s)", root_cache->name, 629 css->serial_nr, memcg_name_buf); 630 if (!cache_name) 631 goto out_unlock; 632 633 s = create_cache(cache_name, root_cache->object_size, 634 root_cache->align, 635 root_cache->flags & CACHE_CREATE_MASK, 636 root_cache->useroffset, root_cache->usersize, 637 root_cache->ctor, memcg, root_cache); 638 /* 639 * If we could not create a memcg cache, do not complain, because 640 * that's not critical at all as we can always proceed with the root 641 * cache. 642 */ 643 if (IS_ERR(s)) { 644 kfree(cache_name); 645 goto out_unlock; 646 } 647 648 /* 649 * Since readers won't lock (see cache_from_memcg_idx()), we need a 650 * barrier here to ensure nobody will see the kmem_cache partially 651 * initialized. 652 */ 653 smp_wmb(); 654 arr->entries[idx] = s; 655 656 out_unlock: 657 mutex_unlock(&slab_mutex); 658 659 put_online_mems(); 660 put_online_cpus(); 661 } 662 663 static void kmemcg_deactivate_workfn(struct work_struct *work) 664 { 665 struct kmem_cache *s = container_of(work, struct kmem_cache, 666 memcg_params.deact_work); 667 668 get_online_cpus(); 669 get_online_mems(); 670 671 mutex_lock(&slab_mutex); 672 673 s->memcg_params.deact_fn(s); 674 675 mutex_unlock(&slab_mutex); 676 677 put_online_mems(); 678 put_online_cpus(); 679 680 /* done, put the ref from slab_deactivate_memcg_cache_rcu_sched() */ 681 css_put(&s->memcg_params.memcg->css); 682 } 683 684 static void kmemcg_deactivate_rcufn(struct rcu_head *head) 685 { 686 struct kmem_cache *s = container_of(head, struct kmem_cache, 687 memcg_params.deact_rcu_head); 688 689 /* 690 * We need to grab blocking locks. Bounce to ->deact_work. The 691 * work item shares the space with the RCU head and can't be 692 * initialized eariler. 693 */ 694 INIT_WORK(&s->memcg_params.deact_work, kmemcg_deactivate_workfn); 695 queue_work(memcg_kmem_cache_wq, &s->memcg_params.deact_work); 696 } 697 698 /** 699 * slab_deactivate_memcg_cache_rcu_sched - schedule deactivation after a 700 * sched RCU grace period 701 * @s: target kmem_cache 702 * @deact_fn: deactivation function to call 703 * 704 * Schedule @deact_fn to be invoked with online cpus, mems and slab_mutex 705 * held after a sched RCU grace period. The slab is guaranteed to stay 706 * alive until @deact_fn is finished. This is to be used from 707 * __kmemcg_cache_deactivate(). 708 */ 709 void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s, 710 void (*deact_fn)(struct kmem_cache *)) 711 { 712 if (WARN_ON_ONCE(is_root_cache(s)) || 713 WARN_ON_ONCE(s->memcg_params.deact_fn)) 714 return; 715 716 if (s->memcg_params.root_cache->memcg_params.dying) 717 return; 718 719 /* pin memcg so that @s doesn't get destroyed in the middle */ 720 css_get(&s->memcg_params.memcg->css); 721 722 s->memcg_params.deact_fn = deact_fn; 723 call_rcu_sched(&s->memcg_params.deact_rcu_head, kmemcg_deactivate_rcufn); 724 } 725 726 void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg) 727 { 728 int idx; 729 struct memcg_cache_array *arr; 730 struct kmem_cache *s, *c; 731 732 idx = memcg_cache_id(memcg); 733 734 get_online_cpus(); 735 get_online_mems(); 736 737 mutex_lock(&slab_mutex); 738 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 739 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 740 lockdep_is_held(&slab_mutex)); 741 c = arr->entries[idx]; 742 if (!c) 743 continue; 744 745 __kmemcg_cache_deactivate(c); 746 arr->entries[idx] = NULL; 747 } 748 mutex_unlock(&slab_mutex); 749 750 put_online_mems(); 751 put_online_cpus(); 752 } 753 754 void memcg_destroy_kmem_caches(struct mem_cgroup *memcg) 755 { 756 struct kmem_cache *s, *s2; 757 758 get_online_cpus(); 759 get_online_mems(); 760 761 mutex_lock(&slab_mutex); 762 list_for_each_entry_safe(s, s2, &memcg->kmem_caches, 763 memcg_params.kmem_caches_node) { 764 /* 765 * The cgroup is about to be freed and therefore has no charges 766 * left. Hence, all its caches must be empty by now. 767 */ 768 BUG_ON(shutdown_cache(s)); 769 } 770 mutex_unlock(&slab_mutex); 771 772 put_online_mems(); 773 put_online_cpus(); 774 } 775 776 static int shutdown_memcg_caches(struct kmem_cache *s) 777 { 778 struct memcg_cache_array *arr; 779 struct kmem_cache *c, *c2; 780 LIST_HEAD(busy); 781 int i; 782 783 BUG_ON(!is_root_cache(s)); 784 785 /* 786 * First, shutdown active caches, i.e. caches that belong to online 787 * memory cgroups. 788 */ 789 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 790 lockdep_is_held(&slab_mutex)); 791 for_each_memcg_cache_index(i) { 792 c = arr->entries[i]; 793 if (!c) 794 continue; 795 if (shutdown_cache(c)) 796 /* 797 * The cache still has objects. Move it to a temporary 798 * list so as not to try to destroy it for a second 799 * time while iterating over inactive caches below. 800 */ 801 list_move(&c->memcg_params.children_node, &busy); 802 else 803 /* 804 * The cache is empty and will be destroyed soon. Clear 805 * the pointer to it in the memcg_caches array so that 806 * it will never be accessed even if the root cache 807 * stays alive. 808 */ 809 arr->entries[i] = NULL; 810 } 811 812 /* 813 * Second, shutdown all caches left from memory cgroups that are now 814 * offline. 815 */ 816 list_for_each_entry_safe(c, c2, &s->memcg_params.children, 817 memcg_params.children_node) 818 shutdown_cache(c); 819 820 list_splice(&busy, &s->memcg_params.children); 821 822 /* 823 * A cache being destroyed must be empty. In particular, this means 824 * that all per memcg caches attached to it must be empty too. 825 */ 826 if (!list_empty(&s->memcg_params.children)) 827 return -EBUSY; 828 return 0; 829 } 830 831 static void flush_memcg_workqueue(struct kmem_cache *s) 832 { 833 mutex_lock(&slab_mutex); 834 s->memcg_params.dying = true; 835 mutex_unlock(&slab_mutex); 836 837 /* 838 * SLUB deactivates the kmem_caches through call_rcu_sched. Make 839 * sure all registered rcu callbacks have been invoked. 840 */ 841 if (IS_ENABLED(CONFIG_SLUB)) 842 rcu_barrier_sched(); 843 844 /* 845 * SLAB and SLUB create memcg kmem_caches through workqueue and SLUB 846 * deactivates the memcg kmem_caches through workqueue. Make sure all 847 * previous workitems on workqueue are processed. 848 */ 849 flush_workqueue(memcg_kmem_cache_wq); 850 } 851 #else 852 static inline int shutdown_memcg_caches(struct kmem_cache *s) 853 { 854 return 0; 855 } 856 857 static inline void flush_memcg_workqueue(struct kmem_cache *s) 858 { 859 } 860 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */ 861 862 void slab_kmem_cache_release(struct kmem_cache *s) 863 { 864 __kmem_cache_release(s); 865 destroy_memcg_params(s); 866 kfree_const(s->name); 867 kmem_cache_free(kmem_cache, s); 868 } 869 870 void kmem_cache_destroy(struct kmem_cache *s) 871 { 872 int err; 873 874 if (unlikely(!s)) 875 return; 876 877 flush_memcg_workqueue(s); 878 879 get_online_cpus(); 880 get_online_mems(); 881 882 mutex_lock(&slab_mutex); 883 884 s->refcount--; 885 if (s->refcount) 886 goto out_unlock; 887 888 err = shutdown_memcg_caches(s); 889 if (!err) 890 err = shutdown_cache(s); 891 892 if (err) { 893 pr_err("kmem_cache_destroy %s: Slab cache still has objects\n", 894 s->name); 895 dump_stack(); 896 } 897 out_unlock: 898 mutex_unlock(&slab_mutex); 899 900 put_online_mems(); 901 put_online_cpus(); 902 } 903 EXPORT_SYMBOL(kmem_cache_destroy); 904 905 /** 906 * kmem_cache_shrink - Shrink a cache. 907 * @cachep: The cache to shrink. 908 * 909 * Releases as many slabs as possible for a cache. 910 * To help debugging, a zero exit status indicates all slabs were released. 911 */ 912 int kmem_cache_shrink(struct kmem_cache *cachep) 913 { 914 int ret; 915 916 get_online_cpus(); 917 get_online_mems(); 918 kasan_cache_shrink(cachep); 919 ret = __kmem_cache_shrink(cachep); 920 put_online_mems(); 921 put_online_cpus(); 922 return ret; 923 } 924 EXPORT_SYMBOL(kmem_cache_shrink); 925 926 bool slab_is_available(void) 927 { 928 return slab_state >= UP; 929 } 930 931 #ifndef CONFIG_SLOB 932 /* Create a cache during boot when no slab services are available yet */ 933 void __init create_boot_cache(struct kmem_cache *s, const char *name, 934 unsigned int size, slab_flags_t flags, 935 unsigned int useroffset, unsigned int usersize) 936 { 937 int err; 938 939 s->name = name; 940 s->size = s->object_size = size; 941 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size); 942 s->useroffset = useroffset; 943 s->usersize = usersize; 944 945 slab_init_memcg_params(s); 946 947 err = __kmem_cache_create(s, flags); 948 949 if (err) 950 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n", 951 name, size, err); 952 953 s->refcount = -1; /* Exempt from merging for now */ 954 } 955 956 struct kmem_cache *__init create_kmalloc_cache(const char *name, 957 unsigned int size, slab_flags_t flags, 958 unsigned int useroffset, unsigned int usersize) 959 { 960 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 961 962 if (!s) 963 panic("Out of memory when creating slab %s\n", name); 964 965 create_boot_cache(s, name, size, flags, useroffset, usersize); 966 list_add(&s->list, &slab_caches); 967 memcg_link_cache(s); 968 s->refcount = 1; 969 return s; 970 } 971 972 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init; 973 EXPORT_SYMBOL(kmalloc_caches); 974 975 #ifdef CONFIG_ZONE_DMA 976 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init; 977 EXPORT_SYMBOL(kmalloc_dma_caches); 978 #endif 979 980 /* 981 * Conversion table for small slabs sizes / 8 to the index in the 982 * kmalloc array. This is necessary for slabs < 192 since we have non power 983 * of two cache sizes there. The size of larger slabs can be determined using 984 * fls. 985 */ 986 static u8 size_index[24] __ro_after_init = { 987 3, /* 8 */ 988 4, /* 16 */ 989 5, /* 24 */ 990 5, /* 32 */ 991 6, /* 40 */ 992 6, /* 48 */ 993 6, /* 56 */ 994 6, /* 64 */ 995 1, /* 72 */ 996 1, /* 80 */ 997 1, /* 88 */ 998 1, /* 96 */ 999 7, /* 104 */ 1000 7, /* 112 */ 1001 7, /* 120 */ 1002 7, /* 128 */ 1003 2, /* 136 */ 1004 2, /* 144 */ 1005 2, /* 152 */ 1006 2, /* 160 */ 1007 2, /* 168 */ 1008 2, /* 176 */ 1009 2, /* 184 */ 1010 2 /* 192 */ 1011 }; 1012 1013 static inline unsigned int size_index_elem(unsigned int bytes) 1014 { 1015 return (bytes - 1) / 8; 1016 } 1017 1018 /* 1019 * Find the kmem_cache structure that serves a given size of 1020 * allocation 1021 */ 1022 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) 1023 { 1024 unsigned int index; 1025 1026 if (unlikely(size > KMALLOC_MAX_SIZE)) { 1027 WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 1028 return NULL; 1029 } 1030 1031 if (size <= 192) { 1032 if (!size) 1033 return ZERO_SIZE_PTR; 1034 1035 index = size_index[size_index_elem(size)]; 1036 } else 1037 index = fls(size - 1); 1038 1039 #ifdef CONFIG_ZONE_DMA 1040 if (unlikely((flags & GFP_DMA))) 1041 return kmalloc_dma_caches[index]; 1042 1043 #endif 1044 return kmalloc_caches[index]; 1045 } 1046 1047 /* 1048 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. 1049 * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is 1050 * kmalloc-67108864. 1051 */ 1052 const struct kmalloc_info_struct kmalloc_info[] __initconst = { 1053 {NULL, 0}, {"kmalloc-96", 96}, 1054 {"kmalloc-192", 192}, {"kmalloc-8", 8}, 1055 {"kmalloc-16", 16}, {"kmalloc-32", 32}, 1056 {"kmalloc-64", 64}, {"kmalloc-128", 128}, 1057 {"kmalloc-256", 256}, {"kmalloc-512", 512}, 1058 {"kmalloc-1024", 1024}, {"kmalloc-2048", 2048}, 1059 {"kmalloc-4096", 4096}, {"kmalloc-8192", 8192}, 1060 {"kmalloc-16384", 16384}, {"kmalloc-32768", 32768}, 1061 {"kmalloc-65536", 65536}, {"kmalloc-131072", 131072}, 1062 {"kmalloc-262144", 262144}, {"kmalloc-524288", 524288}, 1063 {"kmalloc-1048576", 1048576}, {"kmalloc-2097152", 2097152}, 1064 {"kmalloc-4194304", 4194304}, {"kmalloc-8388608", 8388608}, 1065 {"kmalloc-16777216", 16777216}, {"kmalloc-33554432", 33554432}, 1066 {"kmalloc-67108864", 67108864} 1067 }; 1068 1069 /* 1070 * Patch up the size_index table if we have strange large alignment 1071 * requirements for the kmalloc array. This is only the case for 1072 * MIPS it seems. The standard arches will not generate any code here. 1073 * 1074 * Largest permitted alignment is 256 bytes due to the way we 1075 * handle the index determination for the smaller caches. 1076 * 1077 * Make sure that nothing crazy happens if someone starts tinkering 1078 * around with ARCH_KMALLOC_MINALIGN 1079 */ 1080 void __init setup_kmalloc_cache_index_table(void) 1081 { 1082 unsigned int i; 1083 1084 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 1085 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1))); 1086 1087 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) { 1088 unsigned int elem = size_index_elem(i); 1089 1090 if (elem >= ARRAY_SIZE(size_index)) 1091 break; 1092 size_index[elem] = KMALLOC_SHIFT_LOW; 1093 } 1094 1095 if (KMALLOC_MIN_SIZE >= 64) { 1096 /* 1097 * The 96 byte size cache is not used if the alignment 1098 * is 64 byte. 1099 */ 1100 for (i = 64 + 8; i <= 96; i += 8) 1101 size_index[size_index_elem(i)] = 7; 1102 1103 } 1104 1105 if (KMALLOC_MIN_SIZE >= 128) { 1106 /* 1107 * The 192 byte sized cache is not used if the alignment 1108 * is 128 byte. Redirect kmalloc to use the 256 byte cache 1109 * instead. 1110 */ 1111 for (i = 128 + 8; i <= 192; i += 8) 1112 size_index[size_index_elem(i)] = 8; 1113 } 1114 } 1115 1116 static void __init new_kmalloc_cache(int idx, slab_flags_t flags) 1117 { 1118 kmalloc_caches[idx] = create_kmalloc_cache(kmalloc_info[idx].name, 1119 kmalloc_info[idx].size, flags, 0, 1120 kmalloc_info[idx].size); 1121 } 1122 1123 /* 1124 * Create the kmalloc array. Some of the regular kmalloc arrays 1125 * may already have been created because they were needed to 1126 * enable allocations for slab creation. 1127 */ 1128 void __init create_kmalloc_caches(slab_flags_t flags) 1129 { 1130 int i; 1131 1132 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 1133 if (!kmalloc_caches[i]) 1134 new_kmalloc_cache(i, flags); 1135 1136 /* 1137 * Caches that are not of the two-to-the-power-of size. 1138 * These have to be created immediately after the 1139 * earlier power of two caches 1140 */ 1141 if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) 1142 new_kmalloc_cache(1, flags); 1143 if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) 1144 new_kmalloc_cache(2, flags); 1145 } 1146 1147 /* Kmalloc array is now usable */ 1148 slab_state = UP; 1149 1150 #ifdef CONFIG_ZONE_DMA 1151 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) { 1152 struct kmem_cache *s = kmalloc_caches[i]; 1153 1154 if (s) { 1155 unsigned int size = kmalloc_size(i); 1156 char *n = kasprintf(GFP_NOWAIT, 1157 "dma-kmalloc-%u", size); 1158 1159 BUG_ON(!n); 1160 kmalloc_dma_caches[i] = create_kmalloc_cache(n, 1161 size, SLAB_CACHE_DMA | flags, 0, 0); 1162 } 1163 } 1164 #endif 1165 } 1166 #endif /* !CONFIG_SLOB */ 1167 1168 /* 1169 * To avoid unnecessary overhead, we pass through large allocation requests 1170 * directly to the page allocator. We use __GFP_COMP, because we will need to 1171 * know the allocation order to free the pages properly in kfree. 1172 */ 1173 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) 1174 { 1175 void *ret; 1176 struct page *page; 1177 1178 flags |= __GFP_COMP; 1179 page = alloc_pages(flags, order); 1180 ret = page ? page_address(page) : NULL; 1181 kmemleak_alloc(ret, size, 1, flags); 1182 kasan_kmalloc_large(ret, size, flags); 1183 return ret; 1184 } 1185 EXPORT_SYMBOL(kmalloc_order); 1186 1187 #ifdef CONFIG_TRACING 1188 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 1189 { 1190 void *ret = kmalloc_order(size, flags, order); 1191 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 1192 return ret; 1193 } 1194 EXPORT_SYMBOL(kmalloc_order_trace); 1195 #endif 1196 1197 #ifdef CONFIG_SLAB_FREELIST_RANDOM 1198 /* Randomize a generic freelist */ 1199 static void freelist_randomize(struct rnd_state *state, unsigned int *list, 1200 unsigned int count) 1201 { 1202 unsigned int rand; 1203 unsigned int i; 1204 1205 for (i = 0; i < count; i++) 1206 list[i] = i; 1207 1208 /* Fisher-Yates shuffle */ 1209 for (i = count - 1; i > 0; i--) { 1210 rand = prandom_u32_state(state); 1211 rand %= (i + 1); 1212 swap(list[i], list[rand]); 1213 } 1214 } 1215 1216 /* Create a random sequence per cache */ 1217 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 1218 gfp_t gfp) 1219 { 1220 struct rnd_state state; 1221 1222 if (count < 2 || cachep->random_seq) 1223 return 0; 1224 1225 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp); 1226 if (!cachep->random_seq) 1227 return -ENOMEM; 1228 1229 /* Get best entropy at this stage of boot */ 1230 prandom_seed_state(&state, get_random_long()); 1231 1232 freelist_randomize(&state, cachep->random_seq, count); 1233 return 0; 1234 } 1235 1236 /* Destroy the per-cache random freelist sequence */ 1237 void cache_random_seq_destroy(struct kmem_cache *cachep) 1238 { 1239 kfree(cachep->random_seq); 1240 cachep->random_seq = NULL; 1241 } 1242 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 1243 1244 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) 1245 #ifdef CONFIG_SLAB 1246 #define SLABINFO_RIGHTS (0600) 1247 #else 1248 #define SLABINFO_RIGHTS (0400) 1249 #endif 1250 1251 static void print_slabinfo_header(struct seq_file *m) 1252 { 1253 /* 1254 * Output format version, so at least we can change it 1255 * without _too_ many complaints. 1256 */ 1257 #ifdef CONFIG_DEBUG_SLAB 1258 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 1259 #else 1260 seq_puts(m, "slabinfo - version: 2.1\n"); 1261 #endif 1262 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>"); 1263 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 1264 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 1265 #ifdef CONFIG_DEBUG_SLAB 1266 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 1267 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 1268 #endif 1269 seq_putc(m, '\n'); 1270 } 1271 1272 void *slab_start(struct seq_file *m, loff_t *pos) 1273 { 1274 mutex_lock(&slab_mutex); 1275 return seq_list_start(&slab_root_caches, *pos); 1276 } 1277 1278 void *slab_next(struct seq_file *m, void *p, loff_t *pos) 1279 { 1280 return seq_list_next(p, &slab_root_caches, pos); 1281 } 1282 1283 void slab_stop(struct seq_file *m, void *p) 1284 { 1285 mutex_unlock(&slab_mutex); 1286 } 1287 1288 static void 1289 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info) 1290 { 1291 struct kmem_cache *c; 1292 struct slabinfo sinfo; 1293 1294 if (!is_root_cache(s)) 1295 return; 1296 1297 for_each_memcg_cache(c, s) { 1298 memset(&sinfo, 0, sizeof(sinfo)); 1299 get_slabinfo(c, &sinfo); 1300 1301 info->active_slabs += sinfo.active_slabs; 1302 info->num_slabs += sinfo.num_slabs; 1303 info->shared_avail += sinfo.shared_avail; 1304 info->active_objs += sinfo.active_objs; 1305 info->num_objs += sinfo.num_objs; 1306 } 1307 } 1308 1309 static void cache_show(struct kmem_cache *s, struct seq_file *m) 1310 { 1311 struct slabinfo sinfo; 1312 1313 memset(&sinfo, 0, sizeof(sinfo)); 1314 get_slabinfo(s, &sinfo); 1315 1316 memcg_accumulate_slabinfo(s, &sinfo); 1317 1318 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 1319 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, 1320 sinfo.objects_per_slab, (1 << sinfo.cache_order)); 1321 1322 seq_printf(m, " : tunables %4u %4u %4u", 1323 sinfo.limit, sinfo.batchcount, sinfo.shared); 1324 seq_printf(m, " : slabdata %6lu %6lu %6lu", 1325 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); 1326 slabinfo_show_stats(m, s); 1327 seq_putc(m, '\n'); 1328 } 1329 1330 static int slab_show(struct seq_file *m, void *p) 1331 { 1332 struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node); 1333 1334 if (p == slab_root_caches.next) 1335 print_slabinfo_header(m); 1336 cache_show(s, m); 1337 return 0; 1338 } 1339 1340 void dump_unreclaimable_slab(void) 1341 { 1342 struct kmem_cache *s, *s2; 1343 struct slabinfo sinfo; 1344 1345 /* 1346 * Here acquiring slab_mutex is risky since we don't prefer to get 1347 * sleep in oom path. But, without mutex hold, it may introduce a 1348 * risk of crash. 1349 * Use mutex_trylock to protect the list traverse, dump nothing 1350 * without acquiring the mutex. 1351 */ 1352 if (!mutex_trylock(&slab_mutex)) { 1353 pr_warn("excessive unreclaimable slab but cannot dump stats\n"); 1354 return; 1355 } 1356 1357 pr_info("Unreclaimable slab info:\n"); 1358 pr_info("Name Used Total\n"); 1359 1360 list_for_each_entry_safe(s, s2, &slab_caches, list) { 1361 if (!is_root_cache(s) || (s->flags & SLAB_RECLAIM_ACCOUNT)) 1362 continue; 1363 1364 get_slabinfo(s, &sinfo); 1365 1366 if (sinfo.num_objs > 0) 1367 pr_info("%-17s %10luKB %10luKB\n", cache_name(s), 1368 (sinfo.active_objs * s->size) / 1024, 1369 (sinfo.num_objs * s->size) / 1024); 1370 } 1371 mutex_unlock(&slab_mutex); 1372 } 1373 1374 #if defined(CONFIG_MEMCG) 1375 void *memcg_slab_start(struct seq_file *m, loff_t *pos) 1376 { 1377 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1378 1379 mutex_lock(&slab_mutex); 1380 return seq_list_start(&memcg->kmem_caches, *pos); 1381 } 1382 1383 void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos) 1384 { 1385 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1386 1387 return seq_list_next(p, &memcg->kmem_caches, pos); 1388 } 1389 1390 void memcg_slab_stop(struct seq_file *m, void *p) 1391 { 1392 mutex_unlock(&slab_mutex); 1393 } 1394 1395 int memcg_slab_show(struct seq_file *m, void *p) 1396 { 1397 struct kmem_cache *s = list_entry(p, struct kmem_cache, 1398 memcg_params.kmem_caches_node); 1399 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1400 1401 if (p == memcg->kmem_caches.next) 1402 print_slabinfo_header(m); 1403 cache_show(s, m); 1404 return 0; 1405 } 1406 #endif 1407 1408 /* 1409 * slabinfo_op - iterator that generates /proc/slabinfo 1410 * 1411 * Output layout: 1412 * cache-name 1413 * num-active-objs 1414 * total-objs 1415 * object size 1416 * num-active-slabs 1417 * total-slabs 1418 * num-pages-per-slab 1419 * + further values on SMP and with statistics enabled 1420 */ 1421 static const struct seq_operations slabinfo_op = { 1422 .start = slab_start, 1423 .next = slab_next, 1424 .stop = slab_stop, 1425 .show = slab_show, 1426 }; 1427 1428 static int slabinfo_open(struct inode *inode, struct file *file) 1429 { 1430 return seq_open(file, &slabinfo_op); 1431 } 1432 1433 static const struct file_operations proc_slabinfo_operations = { 1434 .open = slabinfo_open, 1435 .read = seq_read, 1436 .write = slabinfo_write, 1437 .llseek = seq_lseek, 1438 .release = seq_release, 1439 }; 1440 1441 static int __init slab_proc_init(void) 1442 { 1443 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, 1444 &proc_slabinfo_operations); 1445 return 0; 1446 } 1447 module_init(slab_proc_init); 1448 #endif /* CONFIG_SLAB || CONFIG_SLUB_DEBUG */ 1449 1450 static __always_inline void *__do_krealloc(const void *p, size_t new_size, 1451 gfp_t flags) 1452 { 1453 void *ret; 1454 size_t ks = 0; 1455 1456 if (p) 1457 ks = ksize(p); 1458 1459 if (ks >= new_size) { 1460 kasan_krealloc((void *)p, new_size, flags); 1461 return (void *)p; 1462 } 1463 1464 ret = kmalloc_track_caller(new_size, flags); 1465 if (ret && p) 1466 memcpy(ret, p, ks); 1467 1468 return ret; 1469 } 1470 1471 /** 1472 * __krealloc - like krealloc() but don't free @p. 1473 * @p: object to reallocate memory for. 1474 * @new_size: how many bytes of memory are required. 1475 * @flags: the type of memory to allocate. 1476 * 1477 * This function is like krealloc() except it never frees the originally 1478 * allocated buffer. Use this if you don't want to free the buffer immediately 1479 * like, for example, with RCU. 1480 */ 1481 void *__krealloc(const void *p, size_t new_size, gfp_t flags) 1482 { 1483 if (unlikely(!new_size)) 1484 return ZERO_SIZE_PTR; 1485 1486 return __do_krealloc(p, new_size, flags); 1487 1488 } 1489 EXPORT_SYMBOL(__krealloc); 1490 1491 /** 1492 * krealloc - reallocate memory. The contents will remain unchanged. 1493 * @p: object to reallocate memory for. 1494 * @new_size: how many bytes of memory are required. 1495 * @flags: the type of memory to allocate. 1496 * 1497 * The contents of the object pointed to are preserved up to the 1498 * lesser of the new and old sizes. If @p is %NULL, krealloc() 1499 * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a 1500 * %NULL pointer, the object pointed to is freed. 1501 */ 1502 void *krealloc(const void *p, size_t new_size, gfp_t flags) 1503 { 1504 void *ret; 1505 1506 if (unlikely(!new_size)) { 1507 kfree(p); 1508 return ZERO_SIZE_PTR; 1509 } 1510 1511 ret = __do_krealloc(p, new_size, flags); 1512 if (ret && p != ret) 1513 kfree(p); 1514 1515 return ret; 1516 } 1517 EXPORT_SYMBOL(krealloc); 1518 1519 /** 1520 * kzfree - like kfree but zero memory 1521 * @p: object to free memory of 1522 * 1523 * The memory of the object @p points to is zeroed before freed. 1524 * If @p is %NULL, kzfree() does nothing. 1525 * 1526 * Note: this function zeroes the whole allocated buffer which can be a good 1527 * deal bigger than the requested buffer size passed to kmalloc(). So be 1528 * careful when using this function in performance sensitive code. 1529 */ 1530 void kzfree(const void *p) 1531 { 1532 size_t ks; 1533 void *mem = (void *)p; 1534 1535 if (unlikely(ZERO_OR_NULL_PTR(mem))) 1536 return; 1537 ks = ksize(mem); 1538 memset(mem, 0, ks); 1539 kfree(mem); 1540 } 1541 EXPORT_SYMBOL(kzfree); 1542 1543 /* Tracepoints definitions. */ 1544 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 1545 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 1546 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); 1547 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); 1548 EXPORT_TRACEPOINT_SYMBOL(kfree); 1549 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1550 1551 int should_failslab(struct kmem_cache *s, gfp_t gfpflags) 1552 { 1553 if (__should_failslab(s, gfpflags)) 1554 return -ENOMEM; 1555 return 0; 1556 } 1557 ALLOW_ERROR_INJECTION(should_failslab, ERRNO); 1558