1 /* 2 * Slab allocator functions that are independent of the allocator strategy 3 * 4 * (C) 2012 Christoph Lameter <cl@linux.com> 5 */ 6 #include <linux/slab.h> 7 8 #include <linux/mm.h> 9 #include <linux/poison.h> 10 #include <linux/interrupt.h> 11 #include <linux/memory.h> 12 #include <linux/compiler.h> 13 #include <linux/module.h> 14 #include <linux/cpu.h> 15 #include <linux/uaccess.h> 16 #include <linux/seq_file.h> 17 #include <linux/proc_fs.h> 18 #include <asm/cacheflush.h> 19 #include <asm/tlbflush.h> 20 #include <asm/page.h> 21 #include <linux/memcontrol.h> 22 23 #define CREATE_TRACE_POINTS 24 #include <trace/events/kmem.h> 25 26 #include "slab.h" 27 28 enum slab_state slab_state; 29 LIST_HEAD(slab_caches); 30 DEFINE_MUTEX(slab_mutex); 31 struct kmem_cache *kmem_cache; 32 33 /* 34 * Set of flags that will prevent slab merging 35 */ 36 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 37 SLAB_TRACE | SLAB_DESTROY_BY_RCU | SLAB_NOLEAKTRACE | \ 38 SLAB_FAILSLAB) 39 40 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ 41 SLAB_NOTRACK | SLAB_ACCOUNT) 42 43 /* 44 * Merge control. If this is set then no merging of slab caches will occur. 45 * (Could be removed. This was introduced to pacify the merge skeptics.) 46 */ 47 static int slab_nomerge; 48 49 static int __init setup_slab_nomerge(char *str) 50 { 51 slab_nomerge = 1; 52 return 1; 53 } 54 55 #ifdef CONFIG_SLUB 56 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0); 57 #endif 58 59 __setup("slab_nomerge", setup_slab_nomerge); 60 61 /* 62 * Determine the size of a slab object 63 */ 64 unsigned int kmem_cache_size(struct kmem_cache *s) 65 { 66 return s->object_size; 67 } 68 EXPORT_SYMBOL(kmem_cache_size); 69 70 #ifdef CONFIG_DEBUG_VM 71 static int kmem_cache_sanity_check(const char *name, size_t size) 72 { 73 struct kmem_cache *s = NULL; 74 75 if (!name || in_interrupt() || size < sizeof(void *) || 76 size > KMALLOC_MAX_SIZE) { 77 pr_err("kmem_cache_create(%s) integrity check failed\n", name); 78 return -EINVAL; 79 } 80 81 list_for_each_entry(s, &slab_caches, list) { 82 char tmp; 83 int res; 84 85 /* 86 * This happens when the module gets unloaded and doesn't 87 * destroy its slab cache and no-one else reuses the vmalloc 88 * area of the module. Print a warning. 89 */ 90 res = probe_kernel_address(s->name, tmp); 91 if (res) { 92 pr_err("Slab cache with size %d has lost its name\n", 93 s->object_size); 94 continue; 95 } 96 } 97 98 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 99 return 0; 100 } 101 #else 102 static inline int kmem_cache_sanity_check(const char *name, size_t size) 103 { 104 return 0; 105 } 106 #endif 107 108 void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 109 { 110 size_t i; 111 112 for (i = 0; i < nr; i++) 113 kmem_cache_free(s, p[i]); 114 } 115 116 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 117 void **p) 118 { 119 size_t i; 120 121 for (i = 0; i < nr; i++) { 122 void *x = p[i] = kmem_cache_alloc(s, flags); 123 if (!x) { 124 __kmem_cache_free_bulk(s, i, p); 125 return 0; 126 } 127 } 128 return i; 129 } 130 131 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 132 void slab_init_memcg_params(struct kmem_cache *s) 133 { 134 s->memcg_params.is_root_cache = true; 135 INIT_LIST_HEAD(&s->memcg_params.list); 136 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); 137 } 138 139 static int init_memcg_params(struct kmem_cache *s, 140 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 141 { 142 struct memcg_cache_array *arr; 143 144 if (memcg) { 145 s->memcg_params.is_root_cache = false; 146 s->memcg_params.memcg = memcg; 147 s->memcg_params.root_cache = root_cache; 148 return 0; 149 } 150 151 slab_init_memcg_params(s); 152 153 if (!memcg_nr_cache_ids) 154 return 0; 155 156 arr = kzalloc(sizeof(struct memcg_cache_array) + 157 memcg_nr_cache_ids * sizeof(void *), 158 GFP_KERNEL); 159 if (!arr) 160 return -ENOMEM; 161 162 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr); 163 return 0; 164 } 165 166 static void destroy_memcg_params(struct kmem_cache *s) 167 { 168 if (is_root_cache(s)) 169 kfree(rcu_access_pointer(s->memcg_params.memcg_caches)); 170 } 171 172 static int update_memcg_params(struct kmem_cache *s, int new_array_size) 173 { 174 struct memcg_cache_array *old, *new; 175 176 if (!is_root_cache(s)) 177 return 0; 178 179 new = kzalloc(sizeof(struct memcg_cache_array) + 180 new_array_size * sizeof(void *), GFP_KERNEL); 181 if (!new) 182 return -ENOMEM; 183 184 old = rcu_dereference_protected(s->memcg_params.memcg_caches, 185 lockdep_is_held(&slab_mutex)); 186 if (old) 187 memcpy(new->entries, old->entries, 188 memcg_nr_cache_ids * sizeof(void *)); 189 190 rcu_assign_pointer(s->memcg_params.memcg_caches, new); 191 if (old) 192 kfree_rcu(old, rcu); 193 return 0; 194 } 195 196 int memcg_update_all_caches(int num_memcgs) 197 { 198 struct kmem_cache *s; 199 int ret = 0; 200 201 mutex_lock(&slab_mutex); 202 list_for_each_entry(s, &slab_caches, list) { 203 ret = update_memcg_params(s, num_memcgs); 204 /* 205 * Instead of freeing the memory, we'll just leave the caches 206 * up to this point in an updated state. 207 */ 208 if (ret) 209 break; 210 } 211 mutex_unlock(&slab_mutex); 212 return ret; 213 } 214 #else 215 static inline int init_memcg_params(struct kmem_cache *s, 216 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 217 { 218 return 0; 219 } 220 221 static inline void destroy_memcg_params(struct kmem_cache *s) 222 { 223 } 224 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */ 225 226 /* 227 * Find a mergeable slab cache 228 */ 229 int slab_unmergeable(struct kmem_cache *s) 230 { 231 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) 232 return 1; 233 234 if (!is_root_cache(s)) 235 return 1; 236 237 if (s->ctor) 238 return 1; 239 240 /* 241 * We may have set a slab to be unmergeable during bootstrap. 242 */ 243 if (s->refcount < 0) 244 return 1; 245 246 return 0; 247 } 248 249 struct kmem_cache *find_mergeable(size_t size, size_t align, 250 unsigned long flags, const char *name, void (*ctor)(void *)) 251 { 252 struct kmem_cache *s; 253 254 if (slab_nomerge || (flags & SLAB_NEVER_MERGE)) 255 return NULL; 256 257 if (ctor) 258 return NULL; 259 260 size = ALIGN(size, sizeof(void *)); 261 align = calculate_alignment(flags, align, size); 262 size = ALIGN(size, align); 263 flags = kmem_cache_flags(size, flags, name, NULL); 264 265 list_for_each_entry_reverse(s, &slab_caches, list) { 266 if (slab_unmergeable(s)) 267 continue; 268 269 if (size > s->size) 270 continue; 271 272 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) 273 continue; 274 /* 275 * Check if alignment is compatible. 276 * Courtesy of Adrian Drzewiecki 277 */ 278 if ((s->size & ~(align - 1)) != s->size) 279 continue; 280 281 if (s->size - size >= sizeof(void *)) 282 continue; 283 284 if (IS_ENABLED(CONFIG_SLAB) && align && 285 (align > s->align || s->align % align)) 286 continue; 287 288 return s; 289 } 290 return NULL; 291 } 292 293 /* 294 * Figure out what the alignment of the objects will be given a set of 295 * flags, a user specified alignment and the size of the objects. 296 */ 297 unsigned long calculate_alignment(unsigned long flags, 298 unsigned long align, unsigned long size) 299 { 300 /* 301 * If the user wants hardware cache aligned objects then follow that 302 * suggestion if the object is sufficiently large. 303 * 304 * The hardware cache alignment cannot override the specified 305 * alignment though. If that is greater then use it. 306 */ 307 if (flags & SLAB_HWCACHE_ALIGN) { 308 unsigned long ralign = cache_line_size(); 309 while (size <= ralign / 2) 310 ralign /= 2; 311 align = max(align, ralign); 312 } 313 314 if (align < ARCH_SLAB_MINALIGN) 315 align = ARCH_SLAB_MINALIGN; 316 317 return ALIGN(align, sizeof(void *)); 318 } 319 320 static struct kmem_cache *create_cache(const char *name, 321 size_t object_size, size_t size, size_t align, 322 unsigned long flags, void (*ctor)(void *), 323 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 324 { 325 struct kmem_cache *s; 326 int err; 327 328 err = -ENOMEM; 329 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 330 if (!s) 331 goto out; 332 333 s->name = name; 334 s->object_size = object_size; 335 s->size = size; 336 s->align = align; 337 s->ctor = ctor; 338 339 err = init_memcg_params(s, memcg, root_cache); 340 if (err) 341 goto out_free_cache; 342 343 err = __kmem_cache_create(s, flags); 344 if (err) 345 goto out_free_cache; 346 347 s->refcount = 1; 348 list_add(&s->list, &slab_caches); 349 out: 350 if (err) 351 return ERR_PTR(err); 352 return s; 353 354 out_free_cache: 355 destroy_memcg_params(s); 356 kmem_cache_free(kmem_cache, s); 357 goto out; 358 } 359 360 /* 361 * kmem_cache_create - Create a cache. 362 * @name: A string which is used in /proc/slabinfo to identify this cache. 363 * @size: The size of objects to be created in this cache. 364 * @align: The required alignment for the objects. 365 * @flags: SLAB flags 366 * @ctor: A constructor for the objects. 367 * 368 * Returns a ptr to the cache on success, NULL on failure. 369 * Cannot be called within a interrupt, but can be interrupted. 370 * The @ctor is run when new pages are allocated by the cache. 371 * 372 * The flags are 373 * 374 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 375 * to catch references to uninitialised memory. 376 * 377 * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check 378 * for buffer overruns. 379 * 380 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 381 * cacheline. This can be beneficial if you're counting cycles as closely 382 * as davem. 383 */ 384 struct kmem_cache * 385 kmem_cache_create(const char *name, size_t size, size_t align, 386 unsigned long flags, void (*ctor)(void *)) 387 { 388 struct kmem_cache *s = NULL; 389 const char *cache_name; 390 int err; 391 392 get_online_cpus(); 393 get_online_mems(); 394 memcg_get_cache_ids(); 395 396 mutex_lock(&slab_mutex); 397 398 err = kmem_cache_sanity_check(name, size); 399 if (err) { 400 goto out_unlock; 401 } 402 403 /* 404 * Some allocators will constraint the set of valid flags to a subset 405 * of all flags. We expect them to define CACHE_CREATE_MASK in this 406 * case, and we'll just provide them with a sanitized version of the 407 * passed flags. 408 */ 409 flags &= CACHE_CREATE_MASK; 410 411 s = __kmem_cache_alias(name, size, align, flags, ctor); 412 if (s) 413 goto out_unlock; 414 415 cache_name = kstrdup_const(name, GFP_KERNEL); 416 if (!cache_name) { 417 err = -ENOMEM; 418 goto out_unlock; 419 } 420 421 s = create_cache(cache_name, size, size, 422 calculate_alignment(flags, align, size), 423 flags, ctor, NULL, NULL); 424 if (IS_ERR(s)) { 425 err = PTR_ERR(s); 426 kfree_const(cache_name); 427 } 428 429 out_unlock: 430 mutex_unlock(&slab_mutex); 431 432 memcg_put_cache_ids(); 433 put_online_mems(); 434 put_online_cpus(); 435 436 if (err) { 437 if (flags & SLAB_PANIC) 438 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", 439 name, err); 440 else { 441 printk(KERN_WARNING "kmem_cache_create(%s) failed with error %d", 442 name, err); 443 dump_stack(); 444 } 445 return NULL; 446 } 447 return s; 448 } 449 EXPORT_SYMBOL(kmem_cache_create); 450 451 static int shutdown_cache(struct kmem_cache *s, 452 struct list_head *release, bool *need_rcu_barrier) 453 { 454 if (__kmem_cache_shutdown(s) != 0) 455 return -EBUSY; 456 457 if (s->flags & SLAB_DESTROY_BY_RCU) 458 *need_rcu_barrier = true; 459 460 list_move(&s->list, release); 461 return 0; 462 } 463 464 static void release_caches(struct list_head *release, bool need_rcu_barrier) 465 { 466 struct kmem_cache *s, *s2; 467 468 if (need_rcu_barrier) 469 rcu_barrier(); 470 471 list_for_each_entry_safe(s, s2, release, list) { 472 #ifdef SLAB_SUPPORTS_SYSFS 473 sysfs_slab_remove(s); 474 #else 475 slab_kmem_cache_release(s); 476 #endif 477 } 478 } 479 480 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 481 /* 482 * memcg_create_kmem_cache - Create a cache for a memory cgroup. 483 * @memcg: The memory cgroup the new cache is for. 484 * @root_cache: The parent of the new cache. 485 * 486 * This function attempts to create a kmem cache that will serve allocation 487 * requests going from @memcg to @root_cache. The new cache inherits properties 488 * from its parent. 489 */ 490 void memcg_create_kmem_cache(struct mem_cgroup *memcg, 491 struct kmem_cache *root_cache) 492 { 493 static char memcg_name_buf[NAME_MAX + 1]; /* protected by slab_mutex */ 494 struct cgroup_subsys_state *css = &memcg->css; 495 struct memcg_cache_array *arr; 496 struct kmem_cache *s = NULL; 497 char *cache_name; 498 int idx; 499 500 get_online_cpus(); 501 get_online_mems(); 502 503 mutex_lock(&slab_mutex); 504 505 /* 506 * The memory cgroup could have been offlined while the cache 507 * creation work was pending. 508 */ 509 if (!memcg_kmem_online(memcg)) 510 goto out_unlock; 511 512 idx = memcg_cache_id(memcg); 513 arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches, 514 lockdep_is_held(&slab_mutex)); 515 516 /* 517 * Since per-memcg caches are created asynchronously on first 518 * allocation (see memcg_kmem_get_cache()), several threads can try to 519 * create the same cache, but only one of them may succeed. 520 */ 521 if (arr->entries[idx]) 522 goto out_unlock; 523 524 cgroup_name(css->cgroup, memcg_name_buf, sizeof(memcg_name_buf)); 525 cache_name = kasprintf(GFP_KERNEL, "%s(%d:%s)", root_cache->name, 526 css->id, memcg_name_buf); 527 if (!cache_name) 528 goto out_unlock; 529 530 s = create_cache(cache_name, root_cache->object_size, 531 root_cache->size, root_cache->align, 532 root_cache->flags, root_cache->ctor, 533 memcg, root_cache); 534 /* 535 * If we could not create a memcg cache, do not complain, because 536 * that's not critical at all as we can always proceed with the root 537 * cache. 538 */ 539 if (IS_ERR(s)) { 540 kfree(cache_name); 541 goto out_unlock; 542 } 543 544 list_add(&s->memcg_params.list, &root_cache->memcg_params.list); 545 546 /* 547 * Since readers won't lock (see cache_from_memcg_idx()), we need a 548 * barrier here to ensure nobody will see the kmem_cache partially 549 * initialized. 550 */ 551 smp_wmb(); 552 arr->entries[idx] = s; 553 554 out_unlock: 555 mutex_unlock(&slab_mutex); 556 557 put_online_mems(); 558 put_online_cpus(); 559 } 560 561 void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg) 562 { 563 int idx; 564 struct memcg_cache_array *arr; 565 struct kmem_cache *s, *c; 566 567 idx = memcg_cache_id(memcg); 568 569 get_online_cpus(); 570 get_online_mems(); 571 572 mutex_lock(&slab_mutex); 573 list_for_each_entry(s, &slab_caches, list) { 574 if (!is_root_cache(s)) 575 continue; 576 577 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 578 lockdep_is_held(&slab_mutex)); 579 c = arr->entries[idx]; 580 if (!c) 581 continue; 582 583 __kmem_cache_shrink(c, true); 584 arr->entries[idx] = NULL; 585 } 586 mutex_unlock(&slab_mutex); 587 588 put_online_mems(); 589 put_online_cpus(); 590 } 591 592 static int __shutdown_memcg_cache(struct kmem_cache *s, 593 struct list_head *release, bool *need_rcu_barrier) 594 { 595 BUG_ON(is_root_cache(s)); 596 597 if (shutdown_cache(s, release, need_rcu_barrier)) 598 return -EBUSY; 599 600 list_del(&s->memcg_params.list); 601 return 0; 602 } 603 604 void memcg_destroy_kmem_caches(struct mem_cgroup *memcg) 605 { 606 LIST_HEAD(release); 607 bool need_rcu_barrier = false; 608 struct kmem_cache *s, *s2; 609 610 get_online_cpus(); 611 get_online_mems(); 612 613 mutex_lock(&slab_mutex); 614 list_for_each_entry_safe(s, s2, &slab_caches, list) { 615 if (is_root_cache(s) || s->memcg_params.memcg != memcg) 616 continue; 617 /* 618 * The cgroup is about to be freed and therefore has no charges 619 * left. Hence, all its caches must be empty by now. 620 */ 621 BUG_ON(__shutdown_memcg_cache(s, &release, &need_rcu_barrier)); 622 } 623 mutex_unlock(&slab_mutex); 624 625 put_online_mems(); 626 put_online_cpus(); 627 628 release_caches(&release, need_rcu_barrier); 629 } 630 631 static int shutdown_memcg_caches(struct kmem_cache *s, 632 struct list_head *release, bool *need_rcu_barrier) 633 { 634 struct memcg_cache_array *arr; 635 struct kmem_cache *c, *c2; 636 LIST_HEAD(busy); 637 int i; 638 639 BUG_ON(!is_root_cache(s)); 640 641 /* 642 * First, shutdown active caches, i.e. caches that belong to online 643 * memory cgroups. 644 */ 645 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 646 lockdep_is_held(&slab_mutex)); 647 for_each_memcg_cache_index(i) { 648 c = arr->entries[i]; 649 if (!c) 650 continue; 651 if (__shutdown_memcg_cache(c, release, need_rcu_barrier)) 652 /* 653 * The cache still has objects. Move it to a temporary 654 * list so as not to try to destroy it for a second 655 * time while iterating over inactive caches below. 656 */ 657 list_move(&c->memcg_params.list, &busy); 658 else 659 /* 660 * The cache is empty and will be destroyed soon. Clear 661 * the pointer to it in the memcg_caches array so that 662 * it will never be accessed even if the root cache 663 * stays alive. 664 */ 665 arr->entries[i] = NULL; 666 } 667 668 /* 669 * Second, shutdown all caches left from memory cgroups that are now 670 * offline. 671 */ 672 list_for_each_entry_safe(c, c2, &s->memcg_params.list, 673 memcg_params.list) 674 __shutdown_memcg_cache(c, release, need_rcu_barrier); 675 676 list_splice(&busy, &s->memcg_params.list); 677 678 /* 679 * A cache being destroyed must be empty. In particular, this means 680 * that all per memcg caches attached to it must be empty too. 681 */ 682 if (!list_empty(&s->memcg_params.list)) 683 return -EBUSY; 684 return 0; 685 } 686 #else 687 static inline int shutdown_memcg_caches(struct kmem_cache *s, 688 struct list_head *release, bool *need_rcu_barrier) 689 { 690 return 0; 691 } 692 #endif /* CONFIG_MEMCG && !CONFIG_SLOB */ 693 694 void slab_kmem_cache_release(struct kmem_cache *s) 695 { 696 destroy_memcg_params(s); 697 kfree_const(s->name); 698 kmem_cache_free(kmem_cache, s); 699 } 700 701 void kmem_cache_destroy(struct kmem_cache *s) 702 { 703 LIST_HEAD(release); 704 bool need_rcu_barrier = false; 705 int err; 706 707 if (unlikely(!s)) 708 return; 709 710 get_online_cpus(); 711 get_online_mems(); 712 713 mutex_lock(&slab_mutex); 714 715 s->refcount--; 716 if (s->refcount) 717 goto out_unlock; 718 719 err = shutdown_memcg_caches(s, &release, &need_rcu_barrier); 720 if (!err) 721 err = shutdown_cache(s, &release, &need_rcu_barrier); 722 723 if (err) { 724 pr_err("kmem_cache_destroy %s: " 725 "Slab cache still has objects\n", s->name); 726 dump_stack(); 727 } 728 out_unlock: 729 mutex_unlock(&slab_mutex); 730 731 put_online_mems(); 732 put_online_cpus(); 733 734 release_caches(&release, need_rcu_barrier); 735 } 736 EXPORT_SYMBOL(kmem_cache_destroy); 737 738 /** 739 * kmem_cache_shrink - Shrink a cache. 740 * @cachep: The cache to shrink. 741 * 742 * Releases as many slabs as possible for a cache. 743 * To help debugging, a zero exit status indicates all slabs were released. 744 */ 745 int kmem_cache_shrink(struct kmem_cache *cachep) 746 { 747 int ret; 748 749 get_online_cpus(); 750 get_online_mems(); 751 ret = __kmem_cache_shrink(cachep, false); 752 put_online_mems(); 753 put_online_cpus(); 754 return ret; 755 } 756 EXPORT_SYMBOL(kmem_cache_shrink); 757 758 bool slab_is_available(void) 759 { 760 return slab_state >= UP; 761 } 762 763 #ifndef CONFIG_SLOB 764 /* Create a cache during boot when no slab services are available yet */ 765 void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size, 766 unsigned long flags) 767 { 768 int err; 769 770 s->name = name; 771 s->size = s->object_size = size; 772 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size); 773 774 slab_init_memcg_params(s); 775 776 err = __kmem_cache_create(s, flags); 777 778 if (err) 779 panic("Creation of kmalloc slab %s size=%zu failed. Reason %d\n", 780 name, size, err); 781 782 s->refcount = -1; /* Exempt from merging for now */ 783 } 784 785 struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size, 786 unsigned long flags) 787 { 788 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 789 790 if (!s) 791 panic("Out of memory when creating slab %s\n", name); 792 793 create_boot_cache(s, name, size, flags); 794 list_add(&s->list, &slab_caches); 795 s->refcount = 1; 796 return s; 797 } 798 799 struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1]; 800 EXPORT_SYMBOL(kmalloc_caches); 801 802 #ifdef CONFIG_ZONE_DMA 803 struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1]; 804 EXPORT_SYMBOL(kmalloc_dma_caches); 805 #endif 806 807 /* 808 * Conversion table for small slabs sizes / 8 to the index in the 809 * kmalloc array. This is necessary for slabs < 192 since we have non power 810 * of two cache sizes there. The size of larger slabs can be determined using 811 * fls. 812 */ 813 static s8 size_index[24] = { 814 3, /* 8 */ 815 4, /* 16 */ 816 5, /* 24 */ 817 5, /* 32 */ 818 6, /* 40 */ 819 6, /* 48 */ 820 6, /* 56 */ 821 6, /* 64 */ 822 1, /* 72 */ 823 1, /* 80 */ 824 1, /* 88 */ 825 1, /* 96 */ 826 7, /* 104 */ 827 7, /* 112 */ 828 7, /* 120 */ 829 7, /* 128 */ 830 2, /* 136 */ 831 2, /* 144 */ 832 2, /* 152 */ 833 2, /* 160 */ 834 2, /* 168 */ 835 2, /* 176 */ 836 2, /* 184 */ 837 2 /* 192 */ 838 }; 839 840 static inline int size_index_elem(size_t bytes) 841 { 842 return (bytes - 1) / 8; 843 } 844 845 /* 846 * Find the kmem_cache structure that serves a given size of 847 * allocation 848 */ 849 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) 850 { 851 int index; 852 853 if (unlikely(size > KMALLOC_MAX_SIZE)) { 854 WARN_ON_ONCE(!(flags & __GFP_NOWARN)); 855 return NULL; 856 } 857 858 if (size <= 192) { 859 if (!size) 860 return ZERO_SIZE_PTR; 861 862 index = size_index[size_index_elem(size)]; 863 } else 864 index = fls(size - 1); 865 866 #ifdef CONFIG_ZONE_DMA 867 if (unlikely((flags & GFP_DMA))) 868 return kmalloc_dma_caches[index]; 869 870 #endif 871 return kmalloc_caches[index]; 872 } 873 874 /* 875 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. 876 * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is 877 * kmalloc-67108864. 878 */ 879 static struct { 880 const char *name; 881 unsigned long size; 882 } const kmalloc_info[] __initconst = { 883 {NULL, 0}, {"kmalloc-96", 96}, 884 {"kmalloc-192", 192}, {"kmalloc-8", 8}, 885 {"kmalloc-16", 16}, {"kmalloc-32", 32}, 886 {"kmalloc-64", 64}, {"kmalloc-128", 128}, 887 {"kmalloc-256", 256}, {"kmalloc-512", 512}, 888 {"kmalloc-1024", 1024}, {"kmalloc-2048", 2048}, 889 {"kmalloc-4096", 4096}, {"kmalloc-8192", 8192}, 890 {"kmalloc-16384", 16384}, {"kmalloc-32768", 32768}, 891 {"kmalloc-65536", 65536}, {"kmalloc-131072", 131072}, 892 {"kmalloc-262144", 262144}, {"kmalloc-524288", 524288}, 893 {"kmalloc-1048576", 1048576}, {"kmalloc-2097152", 2097152}, 894 {"kmalloc-4194304", 4194304}, {"kmalloc-8388608", 8388608}, 895 {"kmalloc-16777216", 16777216}, {"kmalloc-33554432", 33554432}, 896 {"kmalloc-67108864", 67108864} 897 }; 898 899 /* 900 * Patch up the size_index table if we have strange large alignment 901 * requirements for the kmalloc array. This is only the case for 902 * MIPS it seems. The standard arches will not generate any code here. 903 * 904 * Largest permitted alignment is 256 bytes due to the way we 905 * handle the index determination for the smaller caches. 906 * 907 * Make sure that nothing crazy happens if someone starts tinkering 908 * around with ARCH_KMALLOC_MINALIGN 909 */ 910 void __init setup_kmalloc_cache_index_table(void) 911 { 912 int i; 913 914 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 915 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1))); 916 917 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) { 918 int elem = size_index_elem(i); 919 920 if (elem >= ARRAY_SIZE(size_index)) 921 break; 922 size_index[elem] = KMALLOC_SHIFT_LOW; 923 } 924 925 if (KMALLOC_MIN_SIZE >= 64) { 926 /* 927 * The 96 byte size cache is not used if the alignment 928 * is 64 byte. 929 */ 930 for (i = 64 + 8; i <= 96; i += 8) 931 size_index[size_index_elem(i)] = 7; 932 933 } 934 935 if (KMALLOC_MIN_SIZE >= 128) { 936 /* 937 * The 192 byte sized cache is not used if the alignment 938 * is 128 byte. Redirect kmalloc to use the 256 byte cache 939 * instead. 940 */ 941 for (i = 128 + 8; i <= 192; i += 8) 942 size_index[size_index_elem(i)] = 8; 943 } 944 } 945 946 static void __init new_kmalloc_cache(int idx, unsigned long flags) 947 { 948 kmalloc_caches[idx] = create_kmalloc_cache(kmalloc_info[idx].name, 949 kmalloc_info[idx].size, flags); 950 } 951 952 /* 953 * Create the kmalloc array. Some of the regular kmalloc arrays 954 * may already have been created because they were needed to 955 * enable allocations for slab creation. 956 */ 957 void __init create_kmalloc_caches(unsigned long flags) 958 { 959 int i; 960 961 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 962 if (!kmalloc_caches[i]) 963 new_kmalloc_cache(i, flags); 964 965 /* 966 * Caches that are not of the two-to-the-power-of size. 967 * These have to be created immediately after the 968 * earlier power of two caches 969 */ 970 if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6) 971 new_kmalloc_cache(1, flags); 972 if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7) 973 new_kmalloc_cache(2, flags); 974 } 975 976 /* Kmalloc array is now usable */ 977 slab_state = UP; 978 979 #ifdef CONFIG_ZONE_DMA 980 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) { 981 struct kmem_cache *s = kmalloc_caches[i]; 982 983 if (s) { 984 int size = kmalloc_size(i); 985 char *n = kasprintf(GFP_NOWAIT, 986 "dma-kmalloc-%d", size); 987 988 BUG_ON(!n); 989 kmalloc_dma_caches[i] = create_kmalloc_cache(n, 990 size, SLAB_CACHE_DMA | flags); 991 } 992 } 993 #endif 994 } 995 #endif /* !CONFIG_SLOB */ 996 997 /* 998 * To avoid unnecessary overhead, we pass through large allocation requests 999 * directly to the page allocator. We use __GFP_COMP, because we will need to 1000 * know the allocation order to free the pages properly in kfree. 1001 */ 1002 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) 1003 { 1004 void *ret; 1005 struct page *page; 1006 1007 flags |= __GFP_COMP; 1008 page = alloc_kmem_pages(flags, order); 1009 ret = page ? page_address(page) : NULL; 1010 kmemleak_alloc(ret, size, 1, flags); 1011 kasan_kmalloc_large(ret, size); 1012 return ret; 1013 } 1014 EXPORT_SYMBOL(kmalloc_order); 1015 1016 #ifdef CONFIG_TRACING 1017 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 1018 { 1019 void *ret = kmalloc_order(size, flags, order); 1020 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 1021 return ret; 1022 } 1023 EXPORT_SYMBOL(kmalloc_order_trace); 1024 #endif 1025 1026 #ifdef CONFIG_SLABINFO 1027 1028 #ifdef CONFIG_SLAB 1029 #define SLABINFO_RIGHTS (S_IWUSR | S_IRUSR) 1030 #else 1031 #define SLABINFO_RIGHTS S_IRUSR 1032 #endif 1033 1034 static void print_slabinfo_header(struct seq_file *m) 1035 { 1036 /* 1037 * Output format version, so at least we can change it 1038 * without _too_ many complaints. 1039 */ 1040 #ifdef CONFIG_DEBUG_SLAB 1041 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 1042 #else 1043 seq_puts(m, "slabinfo - version: 2.1\n"); 1044 #endif 1045 seq_puts(m, "# name <active_objs> <num_objs> <objsize> " 1046 "<objperslab> <pagesperslab>"); 1047 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 1048 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 1049 #ifdef CONFIG_DEBUG_SLAB 1050 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> " 1051 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 1052 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 1053 #endif 1054 seq_putc(m, '\n'); 1055 } 1056 1057 void *slab_start(struct seq_file *m, loff_t *pos) 1058 { 1059 mutex_lock(&slab_mutex); 1060 return seq_list_start(&slab_caches, *pos); 1061 } 1062 1063 void *slab_next(struct seq_file *m, void *p, loff_t *pos) 1064 { 1065 return seq_list_next(p, &slab_caches, pos); 1066 } 1067 1068 void slab_stop(struct seq_file *m, void *p) 1069 { 1070 mutex_unlock(&slab_mutex); 1071 } 1072 1073 static void 1074 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info) 1075 { 1076 struct kmem_cache *c; 1077 struct slabinfo sinfo; 1078 1079 if (!is_root_cache(s)) 1080 return; 1081 1082 for_each_memcg_cache(c, s) { 1083 memset(&sinfo, 0, sizeof(sinfo)); 1084 get_slabinfo(c, &sinfo); 1085 1086 info->active_slabs += sinfo.active_slabs; 1087 info->num_slabs += sinfo.num_slabs; 1088 info->shared_avail += sinfo.shared_avail; 1089 info->active_objs += sinfo.active_objs; 1090 info->num_objs += sinfo.num_objs; 1091 } 1092 } 1093 1094 static void cache_show(struct kmem_cache *s, struct seq_file *m) 1095 { 1096 struct slabinfo sinfo; 1097 1098 memset(&sinfo, 0, sizeof(sinfo)); 1099 get_slabinfo(s, &sinfo); 1100 1101 memcg_accumulate_slabinfo(s, &sinfo); 1102 1103 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 1104 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, 1105 sinfo.objects_per_slab, (1 << sinfo.cache_order)); 1106 1107 seq_printf(m, " : tunables %4u %4u %4u", 1108 sinfo.limit, sinfo.batchcount, sinfo.shared); 1109 seq_printf(m, " : slabdata %6lu %6lu %6lu", 1110 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); 1111 slabinfo_show_stats(m, s); 1112 seq_putc(m, '\n'); 1113 } 1114 1115 static int slab_show(struct seq_file *m, void *p) 1116 { 1117 struct kmem_cache *s = list_entry(p, struct kmem_cache, list); 1118 1119 if (p == slab_caches.next) 1120 print_slabinfo_header(m); 1121 if (is_root_cache(s)) 1122 cache_show(s, m); 1123 return 0; 1124 } 1125 1126 #if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB) 1127 int memcg_slab_show(struct seq_file *m, void *p) 1128 { 1129 struct kmem_cache *s = list_entry(p, struct kmem_cache, list); 1130 struct mem_cgroup *memcg = mem_cgroup_from_css(seq_css(m)); 1131 1132 if (p == slab_caches.next) 1133 print_slabinfo_header(m); 1134 if (!is_root_cache(s) && s->memcg_params.memcg == memcg) 1135 cache_show(s, m); 1136 return 0; 1137 } 1138 #endif 1139 1140 /* 1141 * slabinfo_op - iterator that generates /proc/slabinfo 1142 * 1143 * Output layout: 1144 * cache-name 1145 * num-active-objs 1146 * total-objs 1147 * object size 1148 * num-active-slabs 1149 * total-slabs 1150 * num-pages-per-slab 1151 * + further values on SMP and with statistics enabled 1152 */ 1153 static const struct seq_operations slabinfo_op = { 1154 .start = slab_start, 1155 .next = slab_next, 1156 .stop = slab_stop, 1157 .show = slab_show, 1158 }; 1159 1160 static int slabinfo_open(struct inode *inode, struct file *file) 1161 { 1162 return seq_open(file, &slabinfo_op); 1163 } 1164 1165 static const struct file_operations proc_slabinfo_operations = { 1166 .open = slabinfo_open, 1167 .read = seq_read, 1168 .write = slabinfo_write, 1169 .llseek = seq_lseek, 1170 .release = seq_release, 1171 }; 1172 1173 static int __init slab_proc_init(void) 1174 { 1175 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, 1176 &proc_slabinfo_operations); 1177 return 0; 1178 } 1179 module_init(slab_proc_init); 1180 #endif /* CONFIG_SLABINFO */ 1181 1182 static __always_inline void *__do_krealloc(const void *p, size_t new_size, 1183 gfp_t flags) 1184 { 1185 void *ret; 1186 size_t ks = 0; 1187 1188 if (p) 1189 ks = ksize(p); 1190 1191 if (ks >= new_size) { 1192 kasan_krealloc((void *)p, new_size); 1193 return (void *)p; 1194 } 1195 1196 ret = kmalloc_track_caller(new_size, flags); 1197 if (ret && p) 1198 memcpy(ret, p, ks); 1199 1200 return ret; 1201 } 1202 1203 /** 1204 * __krealloc - like krealloc() but don't free @p. 1205 * @p: object to reallocate memory for. 1206 * @new_size: how many bytes of memory are required. 1207 * @flags: the type of memory to allocate. 1208 * 1209 * This function is like krealloc() except it never frees the originally 1210 * allocated buffer. Use this if you don't want to free the buffer immediately 1211 * like, for example, with RCU. 1212 */ 1213 void *__krealloc(const void *p, size_t new_size, gfp_t flags) 1214 { 1215 if (unlikely(!new_size)) 1216 return ZERO_SIZE_PTR; 1217 1218 return __do_krealloc(p, new_size, flags); 1219 1220 } 1221 EXPORT_SYMBOL(__krealloc); 1222 1223 /** 1224 * krealloc - reallocate memory. The contents will remain unchanged. 1225 * @p: object to reallocate memory for. 1226 * @new_size: how many bytes of memory are required. 1227 * @flags: the type of memory to allocate. 1228 * 1229 * The contents of the object pointed to are preserved up to the 1230 * lesser of the new and old sizes. If @p is %NULL, krealloc() 1231 * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a 1232 * %NULL pointer, the object pointed to is freed. 1233 */ 1234 void *krealloc(const void *p, size_t new_size, gfp_t flags) 1235 { 1236 void *ret; 1237 1238 if (unlikely(!new_size)) { 1239 kfree(p); 1240 return ZERO_SIZE_PTR; 1241 } 1242 1243 ret = __do_krealloc(p, new_size, flags); 1244 if (ret && p != ret) 1245 kfree(p); 1246 1247 return ret; 1248 } 1249 EXPORT_SYMBOL(krealloc); 1250 1251 /** 1252 * kzfree - like kfree but zero memory 1253 * @p: object to free memory of 1254 * 1255 * The memory of the object @p points to is zeroed before freed. 1256 * If @p is %NULL, kzfree() does nothing. 1257 * 1258 * Note: this function zeroes the whole allocated buffer which can be a good 1259 * deal bigger than the requested buffer size passed to kmalloc(). So be 1260 * careful when using this function in performance sensitive code. 1261 */ 1262 void kzfree(const void *p) 1263 { 1264 size_t ks; 1265 void *mem = (void *)p; 1266 1267 if (unlikely(ZERO_OR_NULL_PTR(mem))) 1268 return; 1269 ks = ksize(mem); 1270 memset(mem, 0, ks); 1271 kfree(mem); 1272 } 1273 EXPORT_SYMBOL(kzfree); 1274 1275 /* Tracepoints definitions. */ 1276 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 1277 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 1278 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); 1279 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); 1280 EXPORT_TRACEPOINT_SYMBOL(kfree); 1281 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1282