1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Slab allocator functions that are independent of the allocator strategy 4 * 5 * (C) 2012 Christoph Lameter <cl@linux.com> 6 */ 7 #include <linux/slab.h> 8 9 #include <linux/mm.h> 10 #include <linux/poison.h> 11 #include <linux/interrupt.h> 12 #include <linux/memory.h> 13 #include <linux/cache.h> 14 #include <linux/compiler.h> 15 #include <linux/module.h> 16 #include <linux/cpu.h> 17 #include <linux/uaccess.h> 18 #include <linux/seq_file.h> 19 #include <linux/proc_fs.h> 20 #include <linux/debugfs.h> 21 #include <asm/cacheflush.h> 22 #include <asm/tlbflush.h> 23 #include <asm/page.h> 24 #include <linux/memcontrol.h> 25 26 #define CREATE_TRACE_POINTS 27 #include <trace/events/kmem.h> 28 29 #include "slab.h" 30 31 enum slab_state slab_state; 32 LIST_HEAD(slab_caches); 33 DEFINE_MUTEX(slab_mutex); 34 struct kmem_cache *kmem_cache; 35 36 #ifdef CONFIG_HARDENED_USERCOPY 37 bool usercopy_fallback __ro_after_init = 38 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK); 39 module_param(usercopy_fallback, bool, 0400); 40 MODULE_PARM_DESC(usercopy_fallback, 41 "WARN instead of reject usercopy whitelist violations"); 42 #endif 43 44 static LIST_HEAD(slab_caches_to_rcu_destroy); 45 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work); 46 static DECLARE_WORK(slab_caches_to_rcu_destroy_work, 47 slab_caches_to_rcu_destroy_workfn); 48 49 /* 50 * Set of flags that will prevent slab merging 51 */ 52 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ 53 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \ 54 SLAB_FAILSLAB | SLAB_KASAN) 55 56 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \ 57 SLAB_CACHE_DMA32 | SLAB_ACCOUNT) 58 59 /* 60 * Merge control. If this is set then no merging of slab caches will occur. 61 */ 62 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT); 63 64 static int __init setup_slab_nomerge(char *str) 65 { 66 slab_nomerge = true; 67 return 1; 68 } 69 70 #ifdef CONFIG_SLUB 71 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0); 72 #endif 73 74 __setup("slab_nomerge", setup_slab_nomerge); 75 76 /* 77 * Determine the size of a slab object 78 */ 79 unsigned int kmem_cache_size(struct kmem_cache *s) 80 { 81 return s->object_size; 82 } 83 EXPORT_SYMBOL(kmem_cache_size); 84 85 #ifdef CONFIG_DEBUG_VM 86 static int kmem_cache_sanity_check(const char *name, unsigned int size) 87 { 88 if (!name || in_interrupt() || size < sizeof(void *) || 89 size > KMALLOC_MAX_SIZE) { 90 pr_err("kmem_cache_create(%s) integrity check failed\n", name); 91 return -EINVAL; 92 } 93 94 WARN_ON(strchr(name, ' ')); /* It confuses parsers */ 95 return 0; 96 } 97 #else 98 static inline int kmem_cache_sanity_check(const char *name, unsigned int size) 99 { 100 return 0; 101 } 102 #endif 103 104 void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p) 105 { 106 size_t i; 107 108 for (i = 0; i < nr; i++) { 109 if (s) 110 kmem_cache_free(s, p[i]); 111 else 112 kfree(p[i]); 113 } 114 } 115 116 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr, 117 void **p) 118 { 119 size_t i; 120 121 for (i = 0; i < nr; i++) { 122 void *x = p[i] = kmem_cache_alloc(s, flags); 123 if (!x) { 124 __kmem_cache_free_bulk(s, i, p); 125 return 0; 126 } 127 } 128 return i; 129 } 130 131 #ifdef CONFIG_MEMCG_KMEM 132 133 LIST_HEAD(slab_root_caches); 134 static DEFINE_SPINLOCK(memcg_kmem_wq_lock); 135 136 static void kmemcg_cache_shutdown(struct percpu_ref *percpu_ref); 137 138 void slab_init_memcg_params(struct kmem_cache *s) 139 { 140 s->memcg_params.root_cache = NULL; 141 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL); 142 INIT_LIST_HEAD(&s->memcg_params.children); 143 s->memcg_params.dying = false; 144 } 145 146 static int init_memcg_params(struct kmem_cache *s, 147 struct kmem_cache *root_cache) 148 { 149 struct memcg_cache_array *arr; 150 151 if (root_cache) { 152 int ret = percpu_ref_init(&s->memcg_params.refcnt, 153 kmemcg_cache_shutdown, 154 0, GFP_KERNEL); 155 if (ret) 156 return ret; 157 158 s->memcg_params.root_cache = root_cache; 159 INIT_LIST_HEAD(&s->memcg_params.children_node); 160 INIT_LIST_HEAD(&s->memcg_params.kmem_caches_node); 161 return 0; 162 } 163 164 slab_init_memcg_params(s); 165 166 if (!memcg_nr_cache_ids) 167 return 0; 168 169 arr = kvzalloc(sizeof(struct memcg_cache_array) + 170 memcg_nr_cache_ids * sizeof(void *), 171 GFP_KERNEL); 172 if (!arr) 173 return -ENOMEM; 174 175 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr); 176 return 0; 177 } 178 179 static void destroy_memcg_params(struct kmem_cache *s) 180 { 181 if (is_root_cache(s)) { 182 kvfree(rcu_access_pointer(s->memcg_params.memcg_caches)); 183 } else { 184 mem_cgroup_put(s->memcg_params.memcg); 185 WRITE_ONCE(s->memcg_params.memcg, NULL); 186 percpu_ref_exit(&s->memcg_params.refcnt); 187 } 188 } 189 190 static void free_memcg_params(struct rcu_head *rcu) 191 { 192 struct memcg_cache_array *old; 193 194 old = container_of(rcu, struct memcg_cache_array, rcu); 195 kvfree(old); 196 } 197 198 static int update_memcg_params(struct kmem_cache *s, int new_array_size) 199 { 200 struct memcg_cache_array *old, *new; 201 202 new = kvzalloc(sizeof(struct memcg_cache_array) + 203 new_array_size * sizeof(void *), GFP_KERNEL); 204 if (!new) 205 return -ENOMEM; 206 207 old = rcu_dereference_protected(s->memcg_params.memcg_caches, 208 lockdep_is_held(&slab_mutex)); 209 if (old) 210 memcpy(new->entries, old->entries, 211 memcg_nr_cache_ids * sizeof(void *)); 212 213 rcu_assign_pointer(s->memcg_params.memcg_caches, new); 214 if (old) 215 call_rcu(&old->rcu, free_memcg_params); 216 return 0; 217 } 218 219 int memcg_update_all_caches(int num_memcgs) 220 { 221 struct kmem_cache *s; 222 int ret = 0; 223 224 mutex_lock(&slab_mutex); 225 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 226 ret = update_memcg_params(s, num_memcgs); 227 /* 228 * Instead of freeing the memory, we'll just leave the caches 229 * up to this point in an updated state. 230 */ 231 if (ret) 232 break; 233 } 234 mutex_unlock(&slab_mutex); 235 return ret; 236 } 237 238 void memcg_link_cache(struct kmem_cache *s, struct mem_cgroup *memcg) 239 { 240 if (is_root_cache(s)) { 241 list_add(&s->root_caches_node, &slab_root_caches); 242 } else { 243 css_get(&memcg->css); 244 s->memcg_params.memcg = memcg; 245 list_add(&s->memcg_params.children_node, 246 &s->memcg_params.root_cache->memcg_params.children); 247 list_add(&s->memcg_params.kmem_caches_node, 248 &s->memcg_params.memcg->kmem_caches); 249 } 250 } 251 252 static void memcg_unlink_cache(struct kmem_cache *s) 253 { 254 if (is_root_cache(s)) { 255 list_del(&s->root_caches_node); 256 } else { 257 list_del(&s->memcg_params.children_node); 258 list_del(&s->memcg_params.kmem_caches_node); 259 } 260 } 261 #else 262 static inline int init_memcg_params(struct kmem_cache *s, 263 struct kmem_cache *root_cache) 264 { 265 return 0; 266 } 267 268 static inline void destroy_memcg_params(struct kmem_cache *s) 269 { 270 } 271 272 static inline void memcg_unlink_cache(struct kmem_cache *s) 273 { 274 } 275 #endif /* CONFIG_MEMCG_KMEM */ 276 277 /* 278 * Figure out what the alignment of the objects will be given a set of 279 * flags, a user specified alignment and the size of the objects. 280 */ 281 static unsigned int calculate_alignment(slab_flags_t flags, 282 unsigned int align, unsigned int size) 283 { 284 /* 285 * If the user wants hardware cache aligned objects then follow that 286 * suggestion if the object is sufficiently large. 287 * 288 * The hardware cache alignment cannot override the specified 289 * alignment though. If that is greater then use it. 290 */ 291 if (flags & SLAB_HWCACHE_ALIGN) { 292 unsigned int ralign; 293 294 ralign = cache_line_size(); 295 while (size <= ralign / 2) 296 ralign /= 2; 297 align = max(align, ralign); 298 } 299 300 if (align < ARCH_SLAB_MINALIGN) 301 align = ARCH_SLAB_MINALIGN; 302 303 return ALIGN(align, sizeof(void *)); 304 } 305 306 /* 307 * Find a mergeable slab cache 308 */ 309 int slab_unmergeable(struct kmem_cache *s) 310 { 311 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE)) 312 return 1; 313 314 if (!is_root_cache(s)) 315 return 1; 316 317 if (s->ctor) 318 return 1; 319 320 if (s->usersize) 321 return 1; 322 323 /* 324 * We may have set a slab to be unmergeable during bootstrap. 325 */ 326 if (s->refcount < 0) 327 return 1; 328 329 return 0; 330 } 331 332 struct kmem_cache *find_mergeable(unsigned int size, unsigned int align, 333 slab_flags_t flags, const char *name, void (*ctor)(void *)) 334 { 335 struct kmem_cache *s; 336 337 if (slab_nomerge) 338 return NULL; 339 340 if (ctor) 341 return NULL; 342 343 size = ALIGN(size, sizeof(void *)); 344 align = calculate_alignment(flags, align, size); 345 size = ALIGN(size, align); 346 flags = kmem_cache_flags(size, flags, name, NULL); 347 348 if (flags & SLAB_NEVER_MERGE) 349 return NULL; 350 351 list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) { 352 if (slab_unmergeable(s)) 353 continue; 354 355 if (size > s->size) 356 continue; 357 358 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME)) 359 continue; 360 /* 361 * Check if alignment is compatible. 362 * Courtesy of Adrian Drzewiecki 363 */ 364 if ((s->size & ~(align - 1)) != s->size) 365 continue; 366 367 if (s->size - size >= sizeof(void *)) 368 continue; 369 370 if (IS_ENABLED(CONFIG_SLAB) && align && 371 (align > s->align || s->align % align)) 372 continue; 373 374 return s; 375 } 376 return NULL; 377 } 378 379 static struct kmem_cache *create_cache(const char *name, 380 unsigned int object_size, unsigned int align, 381 slab_flags_t flags, unsigned int useroffset, 382 unsigned int usersize, void (*ctor)(void *), 383 struct mem_cgroup *memcg, struct kmem_cache *root_cache) 384 { 385 struct kmem_cache *s; 386 int err; 387 388 if (WARN_ON(useroffset + usersize > object_size)) 389 useroffset = usersize = 0; 390 391 err = -ENOMEM; 392 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 393 if (!s) 394 goto out; 395 396 s->name = name; 397 s->size = s->object_size = object_size; 398 s->align = align; 399 s->ctor = ctor; 400 s->useroffset = useroffset; 401 s->usersize = usersize; 402 403 err = init_memcg_params(s, root_cache); 404 if (err) 405 goto out_free_cache; 406 407 err = __kmem_cache_create(s, flags); 408 if (err) 409 goto out_free_cache; 410 411 s->refcount = 1; 412 list_add(&s->list, &slab_caches); 413 memcg_link_cache(s, memcg); 414 out: 415 if (err) 416 return ERR_PTR(err); 417 return s; 418 419 out_free_cache: 420 destroy_memcg_params(s); 421 kmem_cache_free(kmem_cache, s); 422 goto out; 423 } 424 425 /** 426 * kmem_cache_create_usercopy - Create a cache with a region suitable 427 * for copying to userspace 428 * @name: A string which is used in /proc/slabinfo to identify this cache. 429 * @size: The size of objects to be created in this cache. 430 * @align: The required alignment for the objects. 431 * @flags: SLAB flags 432 * @useroffset: Usercopy region offset 433 * @usersize: Usercopy region size 434 * @ctor: A constructor for the objects. 435 * 436 * Cannot be called within a interrupt, but can be interrupted. 437 * The @ctor is run when new pages are allocated by the cache. 438 * 439 * The flags are 440 * 441 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 442 * to catch references to uninitialised memory. 443 * 444 * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check 445 * for buffer overruns. 446 * 447 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 448 * cacheline. This can be beneficial if you're counting cycles as closely 449 * as davem. 450 * 451 * Return: a pointer to the cache on success, NULL on failure. 452 */ 453 struct kmem_cache * 454 kmem_cache_create_usercopy(const char *name, 455 unsigned int size, unsigned int align, 456 slab_flags_t flags, 457 unsigned int useroffset, unsigned int usersize, 458 void (*ctor)(void *)) 459 { 460 struct kmem_cache *s = NULL; 461 const char *cache_name; 462 int err; 463 464 get_online_cpus(); 465 get_online_mems(); 466 memcg_get_cache_ids(); 467 468 mutex_lock(&slab_mutex); 469 470 err = kmem_cache_sanity_check(name, size); 471 if (err) { 472 goto out_unlock; 473 } 474 475 /* Refuse requests with allocator specific flags */ 476 if (flags & ~SLAB_FLAGS_PERMITTED) { 477 err = -EINVAL; 478 goto out_unlock; 479 } 480 481 /* 482 * Some allocators will constraint the set of valid flags to a subset 483 * of all flags. We expect them to define CACHE_CREATE_MASK in this 484 * case, and we'll just provide them with a sanitized version of the 485 * passed flags. 486 */ 487 flags &= CACHE_CREATE_MASK; 488 489 /* Fail closed on bad usersize of useroffset values. */ 490 if (WARN_ON(!usersize && useroffset) || 491 WARN_ON(size < usersize || size - usersize < useroffset)) 492 usersize = useroffset = 0; 493 494 if (!usersize) 495 s = __kmem_cache_alias(name, size, align, flags, ctor); 496 if (s) 497 goto out_unlock; 498 499 cache_name = kstrdup_const(name, GFP_KERNEL); 500 if (!cache_name) { 501 err = -ENOMEM; 502 goto out_unlock; 503 } 504 505 s = create_cache(cache_name, size, 506 calculate_alignment(flags, align, size), 507 flags, useroffset, usersize, ctor, NULL, NULL); 508 if (IS_ERR(s)) { 509 err = PTR_ERR(s); 510 kfree_const(cache_name); 511 } 512 513 out_unlock: 514 mutex_unlock(&slab_mutex); 515 516 memcg_put_cache_ids(); 517 put_online_mems(); 518 put_online_cpus(); 519 520 if (err) { 521 if (flags & SLAB_PANIC) 522 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n", 523 name, err); 524 else { 525 pr_warn("kmem_cache_create(%s) failed with error %d\n", 526 name, err); 527 dump_stack(); 528 } 529 return NULL; 530 } 531 return s; 532 } 533 EXPORT_SYMBOL(kmem_cache_create_usercopy); 534 535 /** 536 * kmem_cache_create - Create a cache. 537 * @name: A string which is used in /proc/slabinfo to identify this cache. 538 * @size: The size of objects to be created in this cache. 539 * @align: The required alignment for the objects. 540 * @flags: SLAB flags 541 * @ctor: A constructor for the objects. 542 * 543 * Cannot be called within a interrupt, but can be interrupted. 544 * The @ctor is run when new pages are allocated by the cache. 545 * 546 * The flags are 547 * 548 * %SLAB_POISON - Poison the slab with a known test pattern (a5a5a5a5) 549 * to catch references to uninitialised memory. 550 * 551 * %SLAB_RED_ZONE - Insert `Red` zones around the allocated memory to check 552 * for buffer overruns. 553 * 554 * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware 555 * cacheline. This can be beneficial if you're counting cycles as closely 556 * as davem. 557 * 558 * Return: a pointer to the cache on success, NULL on failure. 559 */ 560 struct kmem_cache * 561 kmem_cache_create(const char *name, unsigned int size, unsigned int align, 562 slab_flags_t flags, void (*ctor)(void *)) 563 { 564 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0, 565 ctor); 566 } 567 EXPORT_SYMBOL(kmem_cache_create); 568 569 static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work) 570 { 571 LIST_HEAD(to_destroy); 572 struct kmem_cache *s, *s2; 573 574 /* 575 * On destruction, SLAB_TYPESAFE_BY_RCU kmem_caches are put on the 576 * @slab_caches_to_rcu_destroy list. The slab pages are freed 577 * through RCU and and the associated kmem_cache are dereferenced 578 * while freeing the pages, so the kmem_caches should be freed only 579 * after the pending RCU operations are finished. As rcu_barrier() 580 * is a pretty slow operation, we batch all pending destructions 581 * asynchronously. 582 */ 583 mutex_lock(&slab_mutex); 584 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy); 585 mutex_unlock(&slab_mutex); 586 587 if (list_empty(&to_destroy)) 588 return; 589 590 rcu_barrier(); 591 592 list_for_each_entry_safe(s, s2, &to_destroy, list) { 593 #ifdef SLAB_SUPPORTS_SYSFS 594 sysfs_slab_release(s); 595 #else 596 slab_kmem_cache_release(s); 597 #endif 598 } 599 } 600 601 static int shutdown_cache(struct kmem_cache *s) 602 { 603 /* free asan quarantined objects */ 604 kasan_cache_shutdown(s); 605 606 if (__kmem_cache_shutdown(s) != 0) 607 return -EBUSY; 608 609 memcg_unlink_cache(s); 610 list_del(&s->list); 611 612 if (s->flags & SLAB_TYPESAFE_BY_RCU) { 613 #ifdef SLAB_SUPPORTS_SYSFS 614 sysfs_slab_unlink(s); 615 #endif 616 list_add_tail(&s->list, &slab_caches_to_rcu_destroy); 617 schedule_work(&slab_caches_to_rcu_destroy_work); 618 } else { 619 #ifdef SLAB_SUPPORTS_SYSFS 620 sysfs_slab_unlink(s); 621 sysfs_slab_release(s); 622 #else 623 slab_kmem_cache_release(s); 624 #endif 625 } 626 627 return 0; 628 } 629 630 #ifdef CONFIG_MEMCG_KMEM 631 /* 632 * memcg_create_kmem_cache - Create a cache for a memory cgroup. 633 * @memcg: The memory cgroup the new cache is for. 634 * @root_cache: The parent of the new cache. 635 * 636 * This function attempts to create a kmem cache that will serve allocation 637 * requests going from @memcg to @root_cache. The new cache inherits properties 638 * from its parent. 639 */ 640 void memcg_create_kmem_cache(struct mem_cgroup *memcg, 641 struct kmem_cache *root_cache) 642 { 643 static char memcg_name_buf[NAME_MAX + 1]; /* protected by slab_mutex */ 644 struct cgroup_subsys_state *css = &memcg->css; 645 struct memcg_cache_array *arr; 646 struct kmem_cache *s = NULL; 647 char *cache_name; 648 int idx; 649 650 get_online_cpus(); 651 get_online_mems(); 652 653 mutex_lock(&slab_mutex); 654 655 /* 656 * The memory cgroup could have been offlined while the cache 657 * creation work was pending. 658 */ 659 if (memcg->kmem_state != KMEM_ONLINE) 660 goto out_unlock; 661 662 idx = memcg_cache_id(memcg); 663 arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches, 664 lockdep_is_held(&slab_mutex)); 665 666 /* 667 * Since per-memcg caches are created asynchronously on first 668 * allocation (see memcg_kmem_get_cache()), several threads can try to 669 * create the same cache, but only one of them may succeed. 670 */ 671 if (arr->entries[idx]) 672 goto out_unlock; 673 674 cgroup_name(css->cgroup, memcg_name_buf, sizeof(memcg_name_buf)); 675 cache_name = kasprintf(GFP_KERNEL, "%s(%llu:%s)", root_cache->name, 676 css->serial_nr, memcg_name_buf); 677 if (!cache_name) 678 goto out_unlock; 679 680 s = create_cache(cache_name, root_cache->object_size, 681 root_cache->align, 682 root_cache->flags & CACHE_CREATE_MASK, 683 root_cache->useroffset, root_cache->usersize, 684 root_cache->ctor, memcg, root_cache); 685 /* 686 * If we could not create a memcg cache, do not complain, because 687 * that's not critical at all as we can always proceed with the root 688 * cache. 689 */ 690 if (IS_ERR(s)) { 691 kfree(cache_name); 692 goto out_unlock; 693 } 694 695 /* 696 * Since readers won't lock (see memcg_kmem_get_cache()), we need a 697 * barrier here to ensure nobody will see the kmem_cache partially 698 * initialized. 699 */ 700 smp_wmb(); 701 arr->entries[idx] = s; 702 703 out_unlock: 704 mutex_unlock(&slab_mutex); 705 706 put_online_mems(); 707 put_online_cpus(); 708 } 709 710 static void kmemcg_workfn(struct work_struct *work) 711 { 712 struct kmem_cache *s = container_of(work, struct kmem_cache, 713 memcg_params.work); 714 715 get_online_cpus(); 716 get_online_mems(); 717 718 mutex_lock(&slab_mutex); 719 s->memcg_params.work_fn(s); 720 mutex_unlock(&slab_mutex); 721 722 put_online_mems(); 723 put_online_cpus(); 724 } 725 726 static void kmemcg_rcufn(struct rcu_head *head) 727 { 728 struct kmem_cache *s = container_of(head, struct kmem_cache, 729 memcg_params.rcu_head); 730 731 /* 732 * We need to grab blocking locks. Bounce to ->work. The 733 * work item shares the space with the RCU head and can't be 734 * initialized eariler. 735 */ 736 INIT_WORK(&s->memcg_params.work, kmemcg_workfn); 737 queue_work(memcg_kmem_cache_wq, &s->memcg_params.work); 738 } 739 740 static void kmemcg_cache_shutdown_fn(struct kmem_cache *s) 741 { 742 WARN_ON(shutdown_cache(s)); 743 } 744 745 static void kmemcg_cache_shutdown(struct percpu_ref *percpu_ref) 746 { 747 struct kmem_cache *s = container_of(percpu_ref, struct kmem_cache, 748 memcg_params.refcnt); 749 unsigned long flags; 750 751 spin_lock_irqsave(&memcg_kmem_wq_lock, flags); 752 if (s->memcg_params.root_cache->memcg_params.dying) 753 goto unlock; 754 755 s->memcg_params.work_fn = kmemcg_cache_shutdown_fn; 756 INIT_WORK(&s->memcg_params.work, kmemcg_workfn); 757 queue_work(memcg_kmem_cache_wq, &s->memcg_params.work); 758 759 unlock: 760 spin_unlock_irqrestore(&memcg_kmem_wq_lock, flags); 761 } 762 763 static void kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s) 764 { 765 __kmemcg_cache_deactivate_after_rcu(s); 766 percpu_ref_kill(&s->memcg_params.refcnt); 767 } 768 769 static void kmemcg_cache_deactivate(struct kmem_cache *s) 770 { 771 if (WARN_ON_ONCE(is_root_cache(s))) 772 return; 773 774 __kmemcg_cache_deactivate(s); 775 s->flags |= SLAB_DEACTIVATED; 776 777 /* 778 * memcg_kmem_wq_lock is used to synchronize memcg_params.dying 779 * flag and make sure that no new kmem_cache deactivation tasks 780 * are queued (see flush_memcg_workqueue() ). 781 */ 782 spin_lock_irq(&memcg_kmem_wq_lock); 783 if (s->memcg_params.root_cache->memcg_params.dying) 784 goto unlock; 785 786 s->memcg_params.work_fn = kmemcg_cache_deactivate_after_rcu; 787 call_rcu(&s->memcg_params.rcu_head, kmemcg_rcufn); 788 unlock: 789 spin_unlock_irq(&memcg_kmem_wq_lock); 790 } 791 792 void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg, 793 struct mem_cgroup *parent) 794 { 795 int idx; 796 struct memcg_cache_array *arr; 797 struct kmem_cache *s, *c; 798 unsigned int nr_reparented; 799 800 idx = memcg_cache_id(memcg); 801 802 get_online_cpus(); 803 get_online_mems(); 804 805 mutex_lock(&slab_mutex); 806 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 807 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 808 lockdep_is_held(&slab_mutex)); 809 c = arr->entries[idx]; 810 if (!c) 811 continue; 812 813 kmemcg_cache_deactivate(c); 814 arr->entries[idx] = NULL; 815 } 816 nr_reparented = 0; 817 list_for_each_entry(s, &memcg->kmem_caches, 818 memcg_params.kmem_caches_node) { 819 WRITE_ONCE(s->memcg_params.memcg, parent); 820 css_put(&memcg->css); 821 nr_reparented++; 822 } 823 if (nr_reparented) { 824 list_splice_init(&memcg->kmem_caches, 825 &parent->kmem_caches); 826 css_get_many(&parent->css, nr_reparented); 827 } 828 mutex_unlock(&slab_mutex); 829 830 put_online_mems(); 831 put_online_cpus(); 832 } 833 834 static int shutdown_memcg_caches(struct kmem_cache *s) 835 { 836 struct memcg_cache_array *arr; 837 struct kmem_cache *c, *c2; 838 LIST_HEAD(busy); 839 int i; 840 841 BUG_ON(!is_root_cache(s)); 842 843 /* 844 * First, shutdown active caches, i.e. caches that belong to online 845 * memory cgroups. 846 */ 847 arr = rcu_dereference_protected(s->memcg_params.memcg_caches, 848 lockdep_is_held(&slab_mutex)); 849 for_each_memcg_cache_index(i) { 850 c = arr->entries[i]; 851 if (!c) 852 continue; 853 if (shutdown_cache(c)) 854 /* 855 * The cache still has objects. Move it to a temporary 856 * list so as not to try to destroy it for a second 857 * time while iterating over inactive caches below. 858 */ 859 list_move(&c->memcg_params.children_node, &busy); 860 else 861 /* 862 * The cache is empty and will be destroyed soon. Clear 863 * the pointer to it in the memcg_caches array so that 864 * it will never be accessed even if the root cache 865 * stays alive. 866 */ 867 arr->entries[i] = NULL; 868 } 869 870 /* 871 * Second, shutdown all caches left from memory cgroups that are now 872 * offline. 873 */ 874 list_for_each_entry_safe(c, c2, &s->memcg_params.children, 875 memcg_params.children_node) 876 shutdown_cache(c); 877 878 list_splice(&busy, &s->memcg_params.children); 879 880 /* 881 * A cache being destroyed must be empty. In particular, this means 882 * that all per memcg caches attached to it must be empty too. 883 */ 884 if (!list_empty(&s->memcg_params.children)) 885 return -EBUSY; 886 return 0; 887 } 888 889 static void flush_memcg_workqueue(struct kmem_cache *s) 890 { 891 spin_lock_irq(&memcg_kmem_wq_lock); 892 s->memcg_params.dying = true; 893 spin_unlock_irq(&memcg_kmem_wq_lock); 894 895 /* 896 * SLAB and SLUB deactivate the kmem_caches through call_rcu. Make 897 * sure all registered rcu callbacks have been invoked. 898 */ 899 rcu_barrier(); 900 901 /* 902 * SLAB and SLUB create memcg kmem_caches through workqueue and SLUB 903 * deactivates the memcg kmem_caches through workqueue. Make sure all 904 * previous workitems on workqueue are processed. 905 */ 906 flush_workqueue(memcg_kmem_cache_wq); 907 908 /* 909 * If we're racing with children kmem_cache deactivation, it might 910 * take another rcu grace period to complete their destruction. 911 * At this moment the corresponding percpu_ref_kill() call should be 912 * done, but it might take another rcu grace period to complete 913 * switching to the atomic mode. 914 * Please, note that we check without grabbing the slab_mutex. It's safe 915 * because at this moment the children list can't grow. 916 */ 917 if (!list_empty(&s->memcg_params.children)) 918 rcu_barrier(); 919 } 920 #else 921 static inline int shutdown_memcg_caches(struct kmem_cache *s) 922 { 923 return 0; 924 } 925 926 static inline void flush_memcg_workqueue(struct kmem_cache *s) 927 { 928 } 929 #endif /* CONFIG_MEMCG_KMEM */ 930 931 void slab_kmem_cache_release(struct kmem_cache *s) 932 { 933 __kmem_cache_release(s); 934 destroy_memcg_params(s); 935 kfree_const(s->name); 936 kmem_cache_free(kmem_cache, s); 937 } 938 939 void kmem_cache_destroy(struct kmem_cache *s) 940 { 941 int err; 942 943 if (unlikely(!s)) 944 return; 945 946 flush_memcg_workqueue(s); 947 948 get_online_cpus(); 949 get_online_mems(); 950 951 mutex_lock(&slab_mutex); 952 953 s->refcount--; 954 if (s->refcount) 955 goto out_unlock; 956 957 err = shutdown_memcg_caches(s); 958 if (!err) 959 err = shutdown_cache(s); 960 961 if (err) { 962 pr_err("kmem_cache_destroy %s: Slab cache still has objects\n", 963 s->name); 964 dump_stack(); 965 } 966 out_unlock: 967 mutex_unlock(&slab_mutex); 968 969 put_online_mems(); 970 put_online_cpus(); 971 } 972 EXPORT_SYMBOL(kmem_cache_destroy); 973 974 /** 975 * kmem_cache_shrink - Shrink a cache. 976 * @cachep: The cache to shrink. 977 * 978 * Releases as many slabs as possible for a cache. 979 * To help debugging, a zero exit status indicates all slabs were released. 980 * 981 * Return: %0 if all slabs were released, non-zero otherwise 982 */ 983 int kmem_cache_shrink(struct kmem_cache *cachep) 984 { 985 int ret; 986 987 get_online_cpus(); 988 get_online_mems(); 989 kasan_cache_shrink(cachep); 990 ret = __kmem_cache_shrink(cachep); 991 put_online_mems(); 992 put_online_cpus(); 993 return ret; 994 } 995 EXPORT_SYMBOL(kmem_cache_shrink); 996 997 /** 998 * kmem_cache_shrink_all - shrink a cache and all memcg caches for root cache 999 * @s: The cache pointer 1000 */ 1001 void kmem_cache_shrink_all(struct kmem_cache *s) 1002 { 1003 struct kmem_cache *c; 1004 1005 if (!IS_ENABLED(CONFIG_MEMCG_KMEM) || !is_root_cache(s)) { 1006 kmem_cache_shrink(s); 1007 return; 1008 } 1009 1010 get_online_cpus(); 1011 get_online_mems(); 1012 kasan_cache_shrink(s); 1013 __kmem_cache_shrink(s); 1014 1015 /* 1016 * We have to take the slab_mutex to protect from the memcg list 1017 * modification. 1018 */ 1019 mutex_lock(&slab_mutex); 1020 for_each_memcg_cache(c, s) { 1021 /* 1022 * Don't need to shrink deactivated memcg caches. 1023 */ 1024 if (s->flags & SLAB_DEACTIVATED) 1025 continue; 1026 kasan_cache_shrink(c); 1027 __kmem_cache_shrink(c); 1028 } 1029 mutex_unlock(&slab_mutex); 1030 put_online_mems(); 1031 put_online_cpus(); 1032 } 1033 1034 bool slab_is_available(void) 1035 { 1036 return slab_state >= UP; 1037 } 1038 1039 #ifndef CONFIG_SLOB 1040 /* Create a cache during boot when no slab services are available yet */ 1041 void __init create_boot_cache(struct kmem_cache *s, const char *name, 1042 unsigned int size, slab_flags_t flags, 1043 unsigned int useroffset, unsigned int usersize) 1044 { 1045 int err; 1046 unsigned int align = ARCH_KMALLOC_MINALIGN; 1047 1048 s->name = name; 1049 s->size = s->object_size = size; 1050 1051 /* 1052 * For power of two sizes, guarantee natural alignment for kmalloc 1053 * caches, regardless of SL*B debugging options. 1054 */ 1055 if (is_power_of_2(size)) 1056 align = max(align, size); 1057 s->align = calculate_alignment(flags, align, size); 1058 1059 s->useroffset = useroffset; 1060 s->usersize = usersize; 1061 1062 slab_init_memcg_params(s); 1063 1064 err = __kmem_cache_create(s, flags); 1065 1066 if (err) 1067 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n", 1068 name, size, err); 1069 1070 s->refcount = -1; /* Exempt from merging for now */ 1071 } 1072 1073 struct kmem_cache *__init create_kmalloc_cache(const char *name, 1074 unsigned int size, slab_flags_t flags, 1075 unsigned int useroffset, unsigned int usersize) 1076 { 1077 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1078 1079 if (!s) 1080 panic("Out of memory when creating slab %s\n", name); 1081 1082 create_boot_cache(s, name, size, flags, useroffset, usersize); 1083 list_add(&s->list, &slab_caches); 1084 memcg_link_cache(s, NULL); 1085 s->refcount = 1; 1086 return s; 1087 } 1088 1089 struct kmem_cache * 1090 kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init = 1091 { /* initialization for https://bugs.llvm.org/show_bug.cgi?id=42570 */ }; 1092 EXPORT_SYMBOL(kmalloc_caches); 1093 1094 /* 1095 * Conversion table for small slabs sizes / 8 to the index in the 1096 * kmalloc array. This is necessary for slabs < 192 since we have non power 1097 * of two cache sizes there. The size of larger slabs can be determined using 1098 * fls. 1099 */ 1100 static u8 size_index[24] __ro_after_init = { 1101 3, /* 8 */ 1102 4, /* 16 */ 1103 5, /* 24 */ 1104 5, /* 32 */ 1105 6, /* 40 */ 1106 6, /* 48 */ 1107 6, /* 56 */ 1108 6, /* 64 */ 1109 1, /* 72 */ 1110 1, /* 80 */ 1111 1, /* 88 */ 1112 1, /* 96 */ 1113 7, /* 104 */ 1114 7, /* 112 */ 1115 7, /* 120 */ 1116 7, /* 128 */ 1117 2, /* 136 */ 1118 2, /* 144 */ 1119 2, /* 152 */ 1120 2, /* 160 */ 1121 2, /* 168 */ 1122 2, /* 176 */ 1123 2, /* 184 */ 1124 2 /* 192 */ 1125 }; 1126 1127 static inline unsigned int size_index_elem(unsigned int bytes) 1128 { 1129 return (bytes - 1) / 8; 1130 } 1131 1132 /* 1133 * Find the kmem_cache structure that serves a given size of 1134 * allocation 1135 */ 1136 struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags) 1137 { 1138 unsigned int index; 1139 1140 if (size <= 192) { 1141 if (!size) 1142 return ZERO_SIZE_PTR; 1143 1144 index = size_index[size_index_elem(size)]; 1145 } else { 1146 if (WARN_ON_ONCE(size > KMALLOC_MAX_CACHE_SIZE)) 1147 return NULL; 1148 index = fls(size - 1); 1149 } 1150 1151 return kmalloc_caches[kmalloc_type(flags)][index]; 1152 } 1153 1154 #ifdef CONFIG_ZONE_DMA 1155 #define INIT_KMALLOC_INFO(__size, __short_size) \ 1156 { \ 1157 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \ 1158 .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \ 1159 .name[KMALLOC_DMA] = "dma-kmalloc-" #__short_size, \ 1160 .size = __size, \ 1161 } 1162 #else 1163 #define INIT_KMALLOC_INFO(__size, __short_size) \ 1164 { \ 1165 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \ 1166 .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \ 1167 .size = __size, \ 1168 } 1169 #endif 1170 1171 /* 1172 * kmalloc_info[] is to make slub_debug=,kmalloc-xx option work at boot time. 1173 * kmalloc_index() supports up to 2^26=64MB, so the final entry of the table is 1174 * kmalloc-67108864. 1175 */ 1176 const struct kmalloc_info_struct kmalloc_info[] __initconst = { 1177 INIT_KMALLOC_INFO(0, 0), 1178 INIT_KMALLOC_INFO(96, 96), 1179 INIT_KMALLOC_INFO(192, 192), 1180 INIT_KMALLOC_INFO(8, 8), 1181 INIT_KMALLOC_INFO(16, 16), 1182 INIT_KMALLOC_INFO(32, 32), 1183 INIT_KMALLOC_INFO(64, 64), 1184 INIT_KMALLOC_INFO(128, 128), 1185 INIT_KMALLOC_INFO(256, 256), 1186 INIT_KMALLOC_INFO(512, 512), 1187 INIT_KMALLOC_INFO(1024, 1k), 1188 INIT_KMALLOC_INFO(2048, 2k), 1189 INIT_KMALLOC_INFO(4096, 4k), 1190 INIT_KMALLOC_INFO(8192, 8k), 1191 INIT_KMALLOC_INFO(16384, 16k), 1192 INIT_KMALLOC_INFO(32768, 32k), 1193 INIT_KMALLOC_INFO(65536, 64k), 1194 INIT_KMALLOC_INFO(131072, 128k), 1195 INIT_KMALLOC_INFO(262144, 256k), 1196 INIT_KMALLOC_INFO(524288, 512k), 1197 INIT_KMALLOC_INFO(1048576, 1M), 1198 INIT_KMALLOC_INFO(2097152, 2M), 1199 INIT_KMALLOC_INFO(4194304, 4M), 1200 INIT_KMALLOC_INFO(8388608, 8M), 1201 INIT_KMALLOC_INFO(16777216, 16M), 1202 INIT_KMALLOC_INFO(33554432, 32M), 1203 INIT_KMALLOC_INFO(67108864, 64M) 1204 }; 1205 1206 /* 1207 * Patch up the size_index table if we have strange large alignment 1208 * requirements for the kmalloc array. This is only the case for 1209 * MIPS it seems. The standard arches will not generate any code here. 1210 * 1211 * Largest permitted alignment is 256 bytes due to the way we 1212 * handle the index determination for the smaller caches. 1213 * 1214 * Make sure that nothing crazy happens if someone starts tinkering 1215 * around with ARCH_KMALLOC_MINALIGN 1216 */ 1217 void __init setup_kmalloc_cache_index_table(void) 1218 { 1219 unsigned int i; 1220 1221 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 || 1222 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1))); 1223 1224 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) { 1225 unsigned int elem = size_index_elem(i); 1226 1227 if (elem >= ARRAY_SIZE(size_index)) 1228 break; 1229 size_index[elem] = KMALLOC_SHIFT_LOW; 1230 } 1231 1232 if (KMALLOC_MIN_SIZE >= 64) { 1233 /* 1234 * The 96 byte size cache is not used if the alignment 1235 * is 64 byte. 1236 */ 1237 for (i = 64 + 8; i <= 96; i += 8) 1238 size_index[size_index_elem(i)] = 7; 1239 1240 } 1241 1242 if (KMALLOC_MIN_SIZE >= 128) { 1243 /* 1244 * The 192 byte sized cache is not used if the alignment 1245 * is 128 byte. Redirect kmalloc to use the 256 byte cache 1246 * instead. 1247 */ 1248 for (i = 128 + 8; i <= 192; i += 8) 1249 size_index[size_index_elem(i)] = 8; 1250 } 1251 } 1252 1253 static void __init 1254 new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) 1255 { 1256 if (type == KMALLOC_RECLAIM) 1257 flags |= SLAB_RECLAIM_ACCOUNT; 1258 1259 kmalloc_caches[type][idx] = create_kmalloc_cache( 1260 kmalloc_info[idx].name[type], 1261 kmalloc_info[idx].size, flags, 0, 1262 kmalloc_info[idx].size); 1263 } 1264 1265 /* 1266 * Create the kmalloc array. Some of the regular kmalloc arrays 1267 * may already have been created because they were needed to 1268 * enable allocations for slab creation. 1269 */ 1270 void __init create_kmalloc_caches(slab_flags_t flags) 1271 { 1272 int i; 1273 enum kmalloc_cache_type type; 1274 1275 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) { 1276 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { 1277 if (!kmalloc_caches[type][i]) 1278 new_kmalloc_cache(i, type, flags); 1279 1280 /* 1281 * Caches that are not of the two-to-the-power-of size. 1282 * These have to be created immediately after the 1283 * earlier power of two caches 1284 */ 1285 if (KMALLOC_MIN_SIZE <= 32 && i == 6 && 1286 !kmalloc_caches[type][1]) 1287 new_kmalloc_cache(1, type, flags); 1288 if (KMALLOC_MIN_SIZE <= 64 && i == 7 && 1289 !kmalloc_caches[type][2]) 1290 new_kmalloc_cache(2, type, flags); 1291 } 1292 } 1293 1294 /* Kmalloc array is now usable */ 1295 slab_state = UP; 1296 1297 #ifdef CONFIG_ZONE_DMA 1298 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) { 1299 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i]; 1300 1301 if (s) { 1302 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache( 1303 kmalloc_info[i].name[KMALLOC_DMA], 1304 kmalloc_info[i].size, 1305 SLAB_CACHE_DMA | flags, 0, 0); 1306 } 1307 } 1308 #endif 1309 } 1310 #endif /* !CONFIG_SLOB */ 1311 1312 /* 1313 * To avoid unnecessary overhead, we pass through large allocation requests 1314 * directly to the page allocator. We use __GFP_COMP, because we will need to 1315 * know the allocation order to free the pages properly in kfree. 1316 */ 1317 void *kmalloc_order(size_t size, gfp_t flags, unsigned int order) 1318 { 1319 void *ret = NULL; 1320 struct page *page; 1321 1322 flags |= __GFP_COMP; 1323 page = alloc_pages(flags, order); 1324 if (likely(page)) { 1325 ret = page_address(page); 1326 mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE, 1327 1 << order); 1328 } 1329 ret = kasan_kmalloc_large(ret, size, flags); 1330 /* As ret might get tagged, call kmemleak hook after KASAN. */ 1331 kmemleak_alloc(ret, size, 1, flags); 1332 return ret; 1333 } 1334 EXPORT_SYMBOL(kmalloc_order); 1335 1336 #ifdef CONFIG_TRACING 1337 void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order) 1338 { 1339 void *ret = kmalloc_order(size, flags, order); 1340 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags); 1341 return ret; 1342 } 1343 EXPORT_SYMBOL(kmalloc_order_trace); 1344 #endif 1345 1346 #ifdef CONFIG_SLAB_FREELIST_RANDOM 1347 /* Randomize a generic freelist */ 1348 static void freelist_randomize(struct rnd_state *state, unsigned int *list, 1349 unsigned int count) 1350 { 1351 unsigned int rand; 1352 unsigned int i; 1353 1354 for (i = 0; i < count; i++) 1355 list[i] = i; 1356 1357 /* Fisher-Yates shuffle */ 1358 for (i = count - 1; i > 0; i--) { 1359 rand = prandom_u32_state(state); 1360 rand %= (i + 1); 1361 swap(list[i], list[rand]); 1362 } 1363 } 1364 1365 /* Create a random sequence per cache */ 1366 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count, 1367 gfp_t gfp) 1368 { 1369 struct rnd_state state; 1370 1371 if (count < 2 || cachep->random_seq) 1372 return 0; 1373 1374 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp); 1375 if (!cachep->random_seq) 1376 return -ENOMEM; 1377 1378 /* Get best entropy at this stage of boot */ 1379 prandom_seed_state(&state, get_random_long()); 1380 1381 freelist_randomize(&state, cachep->random_seq, count); 1382 return 0; 1383 } 1384 1385 /* Destroy the per-cache random freelist sequence */ 1386 void cache_random_seq_destroy(struct kmem_cache *cachep) 1387 { 1388 kfree(cachep->random_seq); 1389 cachep->random_seq = NULL; 1390 } 1391 #endif /* CONFIG_SLAB_FREELIST_RANDOM */ 1392 1393 #if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG) 1394 #ifdef CONFIG_SLAB 1395 #define SLABINFO_RIGHTS (0600) 1396 #else 1397 #define SLABINFO_RIGHTS (0400) 1398 #endif 1399 1400 static void print_slabinfo_header(struct seq_file *m) 1401 { 1402 /* 1403 * Output format version, so at least we can change it 1404 * without _too_ many complaints. 1405 */ 1406 #ifdef CONFIG_DEBUG_SLAB 1407 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n"); 1408 #else 1409 seq_puts(m, "slabinfo - version: 2.1\n"); 1410 #endif 1411 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>"); 1412 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>"); 1413 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>"); 1414 #ifdef CONFIG_DEBUG_SLAB 1415 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>"); 1416 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>"); 1417 #endif 1418 seq_putc(m, '\n'); 1419 } 1420 1421 void *slab_start(struct seq_file *m, loff_t *pos) 1422 { 1423 mutex_lock(&slab_mutex); 1424 return seq_list_start(&slab_root_caches, *pos); 1425 } 1426 1427 void *slab_next(struct seq_file *m, void *p, loff_t *pos) 1428 { 1429 return seq_list_next(p, &slab_root_caches, pos); 1430 } 1431 1432 void slab_stop(struct seq_file *m, void *p) 1433 { 1434 mutex_unlock(&slab_mutex); 1435 } 1436 1437 static void 1438 memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info) 1439 { 1440 struct kmem_cache *c; 1441 struct slabinfo sinfo; 1442 1443 if (!is_root_cache(s)) 1444 return; 1445 1446 for_each_memcg_cache(c, s) { 1447 memset(&sinfo, 0, sizeof(sinfo)); 1448 get_slabinfo(c, &sinfo); 1449 1450 info->active_slabs += sinfo.active_slabs; 1451 info->num_slabs += sinfo.num_slabs; 1452 info->shared_avail += sinfo.shared_avail; 1453 info->active_objs += sinfo.active_objs; 1454 info->num_objs += sinfo.num_objs; 1455 } 1456 } 1457 1458 static void cache_show(struct kmem_cache *s, struct seq_file *m) 1459 { 1460 struct slabinfo sinfo; 1461 1462 memset(&sinfo, 0, sizeof(sinfo)); 1463 get_slabinfo(s, &sinfo); 1464 1465 memcg_accumulate_slabinfo(s, &sinfo); 1466 1467 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 1468 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size, 1469 sinfo.objects_per_slab, (1 << sinfo.cache_order)); 1470 1471 seq_printf(m, " : tunables %4u %4u %4u", 1472 sinfo.limit, sinfo.batchcount, sinfo.shared); 1473 seq_printf(m, " : slabdata %6lu %6lu %6lu", 1474 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail); 1475 slabinfo_show_stats(m, s); 1476 seq_putc(m, '\n'); 1477 } 1478 1479 static int slab_show(struct seq_file *m, void *p) 1480 { 1481 struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node); 1482 1483 if (p == slab_root_caches.next) 1484 print_slabinfo_header(m); 1485 cache_show(s, m); 1486 return 0; 1487 } 1488 1489 void dump_unreclaimable_slab(void) 1490 { 1491 struct kmem_cache *s, *s2; 1492 struct slabinfo sinfo; 1493 1494 /* 1495 * Here acquiring slab_mutex is risky since we don't prefer to get 1496 * sleep in oom path. But, without mutex hold, it may introduce a 1497 * risk of crash. 1498 * Use mutex_trylock to protect the list traverse, dump nothing 1499 * without acquiring the mutex. 1500 */ 1501 if (!mutex_trylock(&slab_mutex)) { 1502 pr_warn("excessive unreclaimable slab but cannot dump stats\n"); 1503 return; 1504 } 1505 1506 pr_info("Unreclaimable slab info:\n"); 1507 pr_info("Name Used Total\n"); 1508 1509 list_for_each_entry_safe(s, s2, &slab_caches, list) { 1510 if (!is_root_cache(s) || (s->flags & SLAB_RECLAIM_ACCOUNT)) 1511 continue; 1512 1513 get_slabinfo(s, &sinfo); 1514 1515 if (sinfo.num_objs > 0) 1516 pr_info("%-17s %10luKB %10luKB\n", cache_name(s), 1517 (sinfo.active_objs * s->size) / 1024, 1518 (sinfo.num_objs * s->size) / 1024); 1519 } 1520 mutex_unlock(&slab_mutex); 1521 } 1522 1523 #if defined(CONFIG_MEMCG) 1524 void *memcg_slab_start(struct seq_file *m, loff_t *pos) 1525 { 1526 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 1527 1528 mutex_lock(&slab_mutex); 1529 return seq_list_start(&memcg->kmem_caches, *pos); 1530 } 1531 1532 void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos) 1533 { 1534 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 1535 1536 return seq_list_next(p, &memcg->kmem_caches, pos); 1537 } 1538 1539 void memcg_slab_stop(struct seq_file *m, void *p) 1540 { 1541 mutex_unlock(&slab_mutex); 1542 } 1543 1544 int memcg_slab_show(struct seq_file *m, void *p) 1545 { 1546 struct kmem_cache *s = list_entry(p, struct kmem_cache, 1547 memcg_params.kmem_caches_node); 1548 struct mem_cgroup *memcg = mem_cgroup_from_seq(m); 1549 1550 if (p == memcg->kmem_caches.next) 1551 print_slabinfo_header(m); 1552 cache_show(s, m); 1553 return 0; 1554 } 1555 #endif 1556 1557 /* 1558 * slabinfo_op - iterator that generates /proc/slabinfo 1559 * 1560 * Output layout: 1561 * cache-name 1562 * num-active-objs 1563 * total-objs 1564 * object size 1565 * num-active-slabs 1566 * total-slabs 1567 * num-pages-per-slab 1568 * + further values on SMP and with statistics enabled 1569 */ 1570 static const struct seq_operations slabinfo_op = { 1571 .start = slab_start, 1572 .next = slab_next, 1573 .stop = slab_stop, 1574 .show = slab_show, 1575 }; 1576 1577 static int slabinfo_open(struct inode *inode, struct file *file) 1578 { 1579 return seq_open(file, &slabinfo_op); 1580 } 1581 1582 static const struct file_operations proc_slabinfo_operations = { 1583 .open = slabinfo_open, 1584 .read = seq_read, 1585 .write = slabinfo_write, 1586 .llseek = seq_lseek, 1587 .release = seq_release, 1588 }; 1589 1590 static int __init slab_proc_init(void) 1591 { 1592 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, 1593 &proc_slabinfo_operations); 1594 return 0; 1595 } 1596 module_init(slab_proc_init); 1597 1598 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_MEMCG_KMEM) 1599 /* 1600 * Display information about kmem caches that have child memcg caches. 1601 */ 1602 static int memcg_slabinfo_show(struct seq_file *m, void *unused) 1603 { 1604 struct kmem_cache *s, *c; 1605 struct slabinfo sinfo; 1606 1607 mutex_lock(&slab_mutex); 1608 seq_puts(m, "# <name> <css_id[:dead|deact]> <active_objs> <num_objs>"); 1609 seq_puts(m, " <active_slabs> <num_slabs>\n"); 1610 list_for_each_entry(s, &slab_root_caches, root_caches_node) { 1611 /* 1612 * Skip kmem caches that don't have any memcg children. 1613 */ 1614 if (list_empty(&s->memcg_params.children)) 1615 continue; 1616 1617 memset(&sinfo, 0, sizeof(sinfo)); 1618 get_slabinfo(s, &sinfo); 1619 seq_printf(m, "%-17s root %6lu %6lu %6lu %6lu\n", 1620 cache_name(s), sinfo.active_objs, sinfo.num_objs, 1621 sinfo.active_slabs, sinfo.num_slabs); 1622 1623 for_each_memcg_cache(c, s) { 1624 struct cgroup_subsys_state *css; 1625 char *status = ""; 1626 1627 css = &c->memcg_params.memcg->css; 1628 if (!(css->flags & CSS_ONLINE)) 1629 status = ":dead"; 1630 else if (c->flags & SLAB_DEACTIVATED) 1631 status = ":deact"; 1632 1633 memset(&sinfo, 0, sizeof(sinfo)); 1634 get_slabinfo(c, &sinfo); 1635 seq_printf(m, "%-17s %4d%-6s %6lu %6lu %6lu %6lu\n", 1636 cache_name(c), css->id, status, 1637 sinfo.active_objs, sinfo.num_objs, 1638 sinfo.active_slabs, sinfo.num_slabs); 1639 } 1640 } 1641 mutex_unlock(&slab_mutex); 1642 return 0; 1643 } 1644 DEFINE_SHOW_ATTRIBUTE(memcg_slabinfo); 1645 1646 static int __init memcg_slabinfo_init(void) 1647 { 1648 debugfs_create_file("memcg_slabinfo", S_IFREG | S_IRUGO, 1649 NULL, NULL, &memcg_slabinfo_fops); 1650 return 0; 1651 } 1652 1653 late_initcall(memcg_slabinfo_init); 1654 #endif /* CONFIG_DEBUG_FS && CONFIG_MEMCG_KMEM */ 1655 #endif /* CONFIG_SLAB || CONFIG_SLUB_DEBUG */ 1656 1657 static __always_inline void *__do_krealloc(const void *p, size_t new_size, 1658 gfp_t flags) 1659 { 1660 void *ret; 1661 size_t ks = 0; 1662 1663 if (p) 1664 ks = ksize(p); 1665 1666 if (ks >= new_size) { 1667 p = kasan_krealloc((void *)p, new_size, flags); 1668 return (void *)p; 1669 } 1670 1671 ret = kmalloc_track_caller(new_size, flags); 1672 if (ret && p) 1673 memcpy(ret, p, ks); 1674 1675 return ret; 1676 } 1677 1678 /** 1679 * __krealloc - like krealloc() but don't free @p. 1680 * @p: object to reallocate memory for. 1681 * @new_size: how many bytes of memory are required. 1682 * @flags: the type of memory to allocate. 1683 * 1684 * This function is like krealloc() except it never frees the originally 1685 * allocated buffer. Use this if you don't want to free the buffer immediately 1686 * like, for example, with RCU. 1687 * 1688 * Return: pointer to the allocated memory or %NULL in case of error 1689 */ 1690 void *__krealloc(const void *p, size_t new_size, gfp_t flags) 1691 { 1692 if (unlikely(!new_size)) 1693 return ZERO_SIZE_PTR; 1694 1695 return __do_krealloc(p, new_size, flags); 1696 1697 } 1698 EXPORT_SYMBOL(__krealloc); 1699 1700 /** 1701 * krealloc - reallocate memory. The contents will remain unchanged. 1702 * @p: object to reallocate memory for. 1703 * @new_size: how many bytes of memory are required. 1704 * @flags: the type of memory to allocate. 1705 * 1706 * The contents of the object pointed to are preserved up to the 1707 * lesser of the new and old sizes. If @p is %NULL, krealloc() 1708 * behaves exactly like kmalloc(). If @new_size is 0 and @p is not a 1709 * %NULL pointer, the object pointed to is freed. 1710 * 1711 * Return: pointer to the allocated memory or %NULL in case of error 1712 */ 1713 void *krealloc(const void *p, size_t new_size, gfp_t flags) 1714 { 1715 void *ret; 1716 1717 if (unlikely(!new_size)) { 1718 kfree(p); 1719 return ZERO_SIZE_PTR; 1720 } 1721 1722 ret = __do_krealloc(p, new_size, flags); 1723 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret)) 1724 kfree(p); 1725 1726 return ret; 1727 } 1728 EXPORT_SYMBOL(krealloc); 1729 1730 /** 1731 * kzfree - like kfree but zero memory 1732 * @p: object to free memory of 1733 * 1734 * The memory of the object @p points to is zeroed before freed. 1735 * If @p is %NULL, kzfree() does nothing. 1736 * 1737 * Note: this function zeroes the whole allocated buffer which can be a good 1738 * deal bigger than the requested buffer size passed to kmalloc(). So be 1739 * careful when using this function in performance sensitive code. 1740 */ 1741 void kzfree(const void *p) 1742 { 1743 size_t ks; 1744 void *mem = (void *)p; 1745 1746 if (unlikely(ZERO_OR_NULL_PTR(mem))) 1747 return; 1748 ks = ksize(mem); 1749 memset(mem, 0, ks); 1750 kfree(mem); 1751 } 1752 EXPORT_SYMBOL(kzfree); 1753 1754 /** 1755 * ksize - get the actual amount of memory allocated for a given object 1756 * @objp: Pointer to the object 1757 * 1758 * kmalloc may internally round up allocations and return more memory 1759 * than requested. ksize() can be used to determine the actual amount of 1760 * memory allocated. The caller may use this additional memory, even though 1761 * a smaller amount of memory was initially specified with the kmalloc call. 1762 * The caller must guarantee that objp points to a valid object previously 1763 * allocated with either kmalloc() or kmem_cache_alloc(). The object 1764 * must not be freed during the duration of the call. 1765 * 1766 * Return: size of the actual memory used by @objp in bytes 1767 */ 1768 size_t ksize(const void *objp) 1769 { 1770 size_t size; 1771 1772 if (WARN_ON_ONCE(!objp)) 1773 return 0; 1774 /* 1775 * We need to check that the pointed to object is valid, and only then 1776 * unpoison the shadow memory below. We use __kasan_check_read(), to 1777 * generate a more useful report at the time ksize() is called (rather 1778 * than later where behaviour is undefined due to potential 1779 * use-after-free or double-free). 1780 * 1781 * If the pointed to memory is invalid we return 0, to avoid users of 1782 * ksize() writing to and potentially corrupting the memory region. 1783 * 1784 * We want to perform the check before __ksize(), to avoid potentially 1785 * crashing in __ksize() due to accessing invalid metadata. 1786 */ 1787 if (unlikely(objp == ZERO_SIZE_PTR) || !__kasan_check_read(objp, 1)) 1788 return 0; 1789 1790 size = __ksize(objp); 1791 /* 1792 * We assume that ksize callers could use whole allocated area, 1793 * so we need to unpoison this area. 1794 */ 1795 kasan_unpoison_shadow(objp, size); 1796 return size; 1797 } 1798 EXPORT_SYMBOL(ksize); 1799 1800 /* Tracepoints definitions. */ 1801 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 1802 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 1803 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); 1804 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); 1805 EXPORT_TRACEPOINT_SYMBOL(kfree); 1806 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 1807 1808 int should_failslab(struct kmem_cache *s, gfp_t gfpflags) 1809 { 1810 if (__should_failslab(s, gfpflags)) 1811 return -ENOMEM; 1812 return 0; 1813 } 1814 ALLOW_ERROR_INJECTION(should_failslab, ERRNO); 1815