1 /* 2 * SLUB: A slab allocator that limits cache line use instead of queuing 3 * objects in per cpu and per node lists. 4 * 5 * The allocator synchronizes using per slab locks or atomic operatios 6 * and only uses a centralized lock to manage a pool of partial slabs. 7 * 8 * (C) 2007 SGI, Christoph Lameter 9 * (C) 2011 Linux Foundation, Christoph Lameter 10 */ 11 12 #include <linux/mm.h> 13 #include <linux/swap.h> /* struct reclaim_state */ 14 #include <linux/module.h> 15 #include <linux/bit_spinlock.h> 16 #include <linux/interrupt.h> 17 #include <linux/bitops.h> 18 #include <linux/slab.h> 19 #include "slab.h" 20 #include <linux/proc_fs.h> 21 #include <linux/notifier.h> 22 #include <linux/seq_file.h> 23 #include <linux/kasan.h> 24 #include <linux/kmemcheck.h> 25 #include <linux/cpu.h> 26 #include <linux/cpuset.h> 27 #include <linux/mempolicy.h> 28 #include <linux/ctype.h> 29 #include <linux/debugobjects.h> 30 #include <linux/kallsyms.h> 31 #include <linux/memory.h> 32 #include <linux/math64.h> 33 #include <linux/fault-inject.h> 34 #include <linux/stacktrace.h> 35 #include <linux/prefetch.h> 36 #include <linux/memcontrol.h> 37 38 #include <trace/events/kmem.h> 39 40 #include "internal.h" 41 42 /* 43 * Lock order: 44 * 1. slab_mutex (Global Mutex) 45 * 2. node->list_lock 46 * 3. slab_lock(page) (Only on some arches and for debugging) 47 * 48 * slab_mutex 49 * 50 * The role of the slab_mutex is to protect the list of all the slabs 51 * and to synchronize major metadata changes to slab cache structures. 52 * 53 * The slab_lock is only used for debugging and on arches that do not 54 * have the ability to do a cmpxchg_double. It only protects the second 55 * double word in the page struct. Meaning 56 * A. page->freelist -> List of object free in a page 57 * B. page->counters -> Counters of objects 58 * C. page->frozen -> frozen state 59 * 60 * If a slab is frozen then it is exempt from list management. It is not 61 * on any list. The processor that froze the slab is the one who can 62 * perform list operations on the page. Other processors may put objects 63 * onto the freelist but the processor that froze the slab is the only 64 * one that can retrieve the objects from the page's freelist. 65 * 66 * The list_lock protects the partial and full list on each node and 67 * the partial slab counter. If taken then no new slabs may be added or 68 * removed from the lists nor make the number of partial slabs be modified. 69 * (Note that the total number of slabs is an atomic value that may be 70 * modified without taking the list lock). 71 * 72 * The list_lock is a centralized lock and thus we avoid taking it as 73 * much as possible. As long as SLUB does not have to handle partial 74 * slabs, operations can continue without any centralized lock. F.e. 75 * allocating a long series of objects that fill up slabs does not require 76 * the list lock. 77 * Interrupts are disabled during allocation and deallocation in order to 78 * make the slab allocator safe to use in the context of an irq. In addition 79 * interrupts are disabled to ensure that the processor does not change 80 * while handling per_cpu slabs, due to kernel preemption. 81 * 82 * SLUB assigns one slab for allocation to each processor. 83 * Allocations only occur from these slabs called cpu slabs. 84 * 85 * Slabs with free elements are kept on a partial list and during regular 86 * operations no list for full slabs is used. If an object in a full slab is 87 * freed then the slab will show up again on the partial lists. 88 * We track full slabs for debugging purposes though because otherwise we 89 * cannot scan all objects. 90 * 91 * Slabs are freed when they become empty. Teardown and setup is 92 * minimal so we rely on the page allocators per cpu caches for 93 * fast frees and allocs. 94 * 95 * Overloading of page flags that are otherwise used for LRU management. 96 * 97 * PageActive The slab is frozen and exempt from list processing. 98 * This means that the slab is dedicated to a purpose 99 * such as satisfying allocations for a specific 100 * processor. Objects may be freed in the slab while 101 * it is frozen but slab_free will then skip the usual 102 * list operations. It is up to the processor holding 103 * the slab to integrate the slab into the slab lists 104 * when the slab is no longer needed. 105 * 106 * One use of this flag is to mark slabs that are 107 * used for allocations. Then such a slab becomes a cpu 108 * slab. The cpu slab may be equipped with an additional 109 * freelist that allows lockless access to 110 * free objects in addition to the regular freelist 111 * that requires the slab lock. 112 * 113 * PageError Slab requires special handling due to debug 114 * options set. This moves slab handling out of 115 * the fast path and disables lockless freelists. 116 */ 117 118 static inline int kmem_cache_debug(struct kmem_cache *s) 119 { 120 #ifdef CONFIG_SLUB_DEBUG 121 return unlikely(s->flags & SLAB_DEBUG_FLAGS); 122 #else 123 return 0; 124 #endif 125 } 126 127 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 128 { 129 #ifdef CONFIG_SLUB_CPU_PARTIAL 130 return !kmem_cache_debug(s); 131 #else 132 return false; 133 #endif 134 } 135 136 /* 137 * Issues still to be resolved: 138 * 139 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 140 * 141 * - Variable sizing of the per node arrays 142 */ 143 144 /* Enable to test recovery from slab corruption on boot */ 145 #undef SLUB_RESILIENCY_TEST 146 147 /* Enable to log cmpxchg failures */ 148 #undef SLUB_DEBUG_CMPXCHG 149 150 /* 151 * Mininum number of partial slabs. These will be left on the partial 152 * lists even if they are empty. kmem_cache_shrink may reclaim them. 153 */ 154 #define MIN_PARTIAL 5 155 156 /* 157 * Maximum number of desirable partial slabs. 158 * The existence of more partial slabs makes kmem_cache_shrink 159 * sort the partial list by the number of objects in use. 160 */ 161 #define MAX_PARTIAL 10 162 163 #define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ 164 SLAB_POISON | SLAB_STORE_USER) 165 166 /* 167 * Debugging flags that require metadata to be stored in the slab. These get 168 * disabled when slub_debug=O is used and a cache's min order increases with 169 * metadata. 170 */ 171 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 172 173 #define OO_SHIFT 16 174 #define OO_MASK ((1 << OO_SHIFT) - 1) 175 #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ 176 177 /* Internal SLUB flags */ 178 #define __OBJECT_POISON 0x80000000UL /* Poison object */ 179 #define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ 180 181 #ifdef CONFIG_SMP 182 static struct notifier_block slab_notifier; 183 #endif 184 185 /* 186 * Tracking user of a slab. 187 */ 188 #define TRACK_ADDRS_COUNT 16 189 struct track { 190 unsigned long addr; /* Called from address */ 191 #ifdef CONFIG_STACKTRACE 192 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ 193 #endif 194 int cpu; /* Was running on cpu */ 195 int pid; /* Pid context */ 196 unsigned long when; /* When did the operation occur */ 197 }; 198 199 enum track_item { TRACK_ALLOC, TRACK_FREE }; 200 201 #ifdef CONFIG_SYSFS 202 static int sysfs_slab_add(struct kmem_cache *); 203 static int sysfs_slab_alias(struct kmem_cache *, const char *); 204 static void memcg_propagate_slab_attrs(struct kmem_cache *s); 205 #else 206 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 207 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 208 { return 0; } 209 static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } 210 #endif 211 212 static inline void stat(const struct kmem_cache *s, enum stat_item si) 213 { 214 #ifdef CONFIG_SLUB_STATS 215 /* 216 * The rmw is racy on a preemptible kernel but this is acceptable, so 217 * avoid this_cpu_add()'s irq-disable overhead. 218 */ 219 raw_cpu_inc(s->cpu_slab->stat[si]); 220 #endif 221 } 222 223 /******************************************************************** 224 * Core slab cache functions 225 *******************************************************************/ 226 227 /* Verify that a pointer has an address that is valid within a slab page */ 228 static inline int check_valid_pointer(struct kmem_cache *s, 229 struct page *page, const void *object) 230 { 231 void *base; 232 233 if (!object) 234 return 1; 235 236 base = page_address(page); 237 if (object < base || object >= base + page->objects * s->size || 238 (object - base) % s->size) { 239 return 0; 240 } 241 242 return 1; 243 } 244 245 static inline void *get_freepointer(struct kmem_cache *s, void *object) 246 { 247 return *(void **)(object + s->offset); 248 } 249 250 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 251 { 252 prefetch(object + s->offset); 253 } 254 255 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) 256 { 257 void *p; 258 259 #ifdef CONFIG_DEBUG_PAGEALLOC 260 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); 261 #else 262 p = get_freepointer(s, object); 263 #endif 264 return p; 265 } 266 267 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 268 { 269 *(void **)(object + s->offset) = fp; 270 } 271 272 /* Loop over all objects in a slab */ 273 #define for_each_object(__p, __s, __addr, __objects) \ 274 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ 275 __p += (__s)->size) 276 277 #define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ 278 for (__p = (__addr), __idx = 1; __idx <= __objects;\ 279 __p += (__s)->size, __idx++) 280 281 /* Determine object index from a given position */ 282 static inline int slab_index(void *p, struct kmem_cache *s, void *addr) 283 { 284 return (p - addr) / s->size; 285 } 286 287 static inline size_t slab_ksize(const struct kmem_cache *s) 288 { 289 #ifdef CONFIG_SLUB_DEBUG 290 /* 291 * Debugging requires use of the padding between object 292 * and whatever may come after it. 293 */ 294 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 295 return s->object_size; 296 297 #endif 298 /* 299 * If we have the need to store the freelist pointer 300 * back there or track user information then we can 301 * only use the space before that information. 302 */ 303 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) 304 return s->inuse; 305 /* 306 * Else we can use all the padding etc for the allocation 307 */ 308 return s->size; 309 } 310 311 static inline int order_objects(int order, unsigned long size, int reserved) 312 { 313 return ((PAGE_SIZE << order) - reserved) / size; 314 } 315 316 static inline struct kmem_cache_order_objects oo_make(int order, 317 unsigned long size, int reserved) 318 { 319 struct kmem_cache_order_objects x = { 320 (order << OO_SHIFT) + order_objects(order, size, reserved) 321 }; 322 323 return x; 324 } 325 326 static inline int oo_order(struct kmem_cache_order_objects x) 327 { 328 return x.x >> OO_SHIFT; 329 } 330 331 static inline int oo_objects(struct kmem_cache_order_objects x) 332 { 333 return x.x & OO_MASK; 334 } 335 336 /* 337 * Per slab locking using the pagelock 338 */ 339 static __always_inline void slab_lock(struct page *page) 340 { 341 bit_spin_lock(PG_locked, &page->flags); 342 } 343 344 static __always_inline void slab_unlock(struct page *page) 345 { 346 __bit_spin_unlock(PG_locked, &page->flags); 347 } 348 349 static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) 350 { 351 struct page tmp; 352 tmp.counters = counters_new; 353 /* 354 * page->counters can cover frozen/inuse/objects as well 355 * as page->_count. If we assign to ->counters directly 356 * we run the risk of losing updates to page->_count, so 357 * be careful and only assign to the fields we need. 358 */ 359 page->frozen = tmp.frozen; 360 page->inuse = tmp.inuse; 361 page->objects = tmp.objects; 362 } 363 364 /* Interrupts must be disabled (for the fallback code to work right) */ 365 static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, 366 void *freelist_old, unsigned long counters_old, 367 void *freelist_new, unsigned long counters_new, 368 const char *n) 369 { 370 VM_BUG_ON(!irqs_disabled()); 371 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 372 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 373 if (s->flags & __CMPXCHG_DOUBLE) { 374 if (cmpxchg_double(&page->freelist, &page->counters, 375 freelist_old, counters_old, 376 freelist_new, counters_new)) 377 return 1; 378 } else 379 #endif 380 { 381 slab_lock(page); 382 if (page->freelist == freelist_old && 383 page->counters == counters_old) { 384 page->freelist = freelist_new; 385 set_page_slub_counters(page, counters_new); 386 slab_unlock(page); 387 return 1; 388 } 389 slab_unlock(page); 390 } 391 392 cpu_relax(); 393 stat(s, CMPXCHG_DOUBLE_FAIL); 394 395 #ifdef SLUB_DEBUG_CMPXCHG 396 pr_info("%s %s: cmpxchg double redo ", n, s->name); 397 #endif 398 399 return 0; 400 } 401 402 static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, 403 void *freelist_old, unsigned long counters_old, 404 void *freelist_new, unsigned long counters_new, 405 const char *n) 406 { 407 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 408 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 409 if (s->flags & __CMPXCHG_DOUBLE) { 410 if (cmpxchg_double(&page->freelist, &page->counters, 411 freelist_old, counters_old, 412 freelist_new, counters_new)) 413 return 1; 414 } else 415 #endif 416 { 417 unsigned long flags; 418 419 local_irq_save(flags); 420 slab_lock(page); 421 if (page->freelist == freelist_old && 422 page->counters == counters_old) { 423 page->freelist = freelist_new; 424 set_page_slub_counters(page, counters_new); 425 slab_unlock(page); 426 local_irq_restore(flags); 427 return 1; 428 } 429 slab_unlock(page); 430 local_irq_restore(flags); 431 } 432 433 cpu_relax(); 434 stat(s, CMPXCHG_DOUBLE_FAIL); 435 436 #ifdef SLUB_DEBUG_CMPXCHG 437 pr_info("%s %s: cmpxchg double redo ", n, s->name); 438 #endif 439 440 return 0; 441 } 442 443 #ifdef CONFIG_SLUB_DEBUG 444 /* 445 * Determine a map of object in use on a page. 446 * 447 * Node listlock must be held to guarantee that the page does 448 * not vanish from under us. 449 */ 450 static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) 451 { 452 void *p; 453 void *addr = page_address(page); 454 455 for (p = page->freelist; p; p = get_freepointer(s, p)) 456 set_bit(slab_index(p, s, addr), map); 457 } 458 459 /* 460 * Debug settings: 461 */ 462 #ifdef CONFIG_SLUB_DEBUG_ON 463 static int slub_debug = DEBUG_DEFAULT_FLAGS; 464 #else 465 static int slub_debug; 466 #endif 467 468 static char *slub_debug_slabs; 469 static int disable_higher_order_debug; 470 471 /* 472 * slub is about to manipulate internal object metadata. This memory lies 473 * outside the range of the allocated object, so accessing it would normally 474 * be reported by kasan as a bounds error. metadata_access_enable() is used 475 * to tell kasan that these accesses are OK. 476 */ 477 static inline void metadata_access_enable(void) 478 { 479 kasan_disable_current(); 480 } 481 482 static inline void metadata_access_disable(void) 483 { 484 kasan_enable_current(); 485 } 486 487 /* 488 * Object debugging 489 */ 490 static void print_section(char *text, u8 *addr, unsigned int length) 491 { 492 metadata_access_enable(); 493 print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, 494 length, 1); 495 metadata_access_disable(); 496 } 497 498 static struct track *get_track(struct kmem_cache *s, void *object, 499 enum track_item alloc) 500 { 501 struct track *p; 502 503 if (s->offset) 504 p = object + s->offset + sizeof(void *); 505 else 506 p = object + s->inuse; 507 508 return p + alloc; 509 } 510 511 static void set_track(struct kmem_cache *s, void *object, 512 enum track_item alloc, unsigned long addr) 513 { 514 struct track *p = get_track(s, object, alloc); 515 516 if (addr) { 517 #ifdef CONFIG_STACKTRACE 518 struct stack_trace trace; 519 int i; 520 521 trace.nr_entries = 0; 522 trace.max_entries = TRACK_ADDRS_COUNT; 523 trace.entries = p->addrs; 524 trace.skip = 3; 525 metadata_access_enable(); 526 save_stack_trace(&trace); 527 metadata_access_disable(); 528 529 /* See rant in lockdep.c */ 530 if (trace.nr_entries != 0 && 531 trace.entries[trace.nr_entries - 1] == ULONG_MAX) 532 trace.nr_entries--; 533 534 for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) 535 p->addrs[i] = 0; 536 #endif 537 p->addr = addr; 538 p->cpu = smp_processor_id(); 539 p->pid = current->pid; 540 p->when = jiffies; 541 } else 542 memset(p, 0, sizeof(struct track)); 543 } 544 545 static void init_tracking(struct kmem_cache *s, void *object) 546 { 547 if (!(s->flags & SLAB_STORE_USER)) 548 return; 549 550 set_track(s, object, TRACK_FREE, 0UL); 551 set_track(s, object, TRACK_ALLOC, 0UL); 552 } 553 554 static void print_track(const char *s, struct track *t) 555 { 556 if (!t->addr) 557 return; 558 559 pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n", 560 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); 561 #ifdef CONFIG_STACKTRACE 562 { 563 int i; 564 for (i = 0; i < TRACK_ADDRS_COUNT; i++) 565 if (t->addrs[i]) 566 pr_err("\t%pS\n", (void *)t->addrs[i]); 567 else 568 break; 569 } 570 #endif 571 } 572 573 static void print_tracking(struct kmem_cache *s, void *object) 574 { 575 if (!(s->flags & SLAB_STORE_USER)) 576 return; 577 578 print_track("Allocated", get_track(s, object, TRACK_ALLOC)); 579 print_track("Freed", get_track(s, object, TRACK_FREE)); 580 } 581 582 static void print_page_info(struct page *page) 583 { 584 pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", 585 page, page->objects, page->inuse, page->freelist, page->flags); 586 587 } 588 589 static void slab_bug(struct kmem_cache *s, char *fmt, ...) 590 { 591 struct va_format vaf; 592 va_list args; 593 594 va_start(args, fmt); 595 vaf.fmt = fmt; 596 vaf.va = &args; 597 pr_err("=============================================================================\n"); 598 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); 599 pr_err("-----------------------------------------------------------------------------\n\n"); 600 601 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 602 va_end(args); 603 } 604 605 static void slab_fix(struct kmem_cache *s, char *fmt, ...) 606 { 607 struct va_format vaf; 608 va_list args; 609 610 va_start(args, fmt); 611 vaf.fmt = fmt; 612 vaf.va = &args; 613 pr_err("FIX %s: %pV\n", s->name, &vaf); 614 va_end(args); 615 } 616 617 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) 618 { 619 unsigned int off; /* Offset of last byte */ 620 u8 *addr = page_address(page); 621 622 print_tracking(s, p); 623 624 print_page_info(page); 625 626 pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", 627 p, p - addr, get_freepointer(s, p)); 628 629 if (p > addr + 16) 630 print_section("Bytes b4 ", p - 16, 16); 631 632 print_section("Object ", p, min_t(unsigned long, s->object_size, 633 PAGE_SIZE)); 634 if (s->flags & SLAB_RED_ZONE) 635 print_section("Redzone ", p + s->object_size, 636 s->inuse - s->object_size); 637 638 if (s->offset) 639 off = s->offset + sizeof(void *); 640 else 641 off = s->inuse; 642 643 if (s->flags & SLAB_STORE_USER) 644 off += 2 * sizeof(struct track); 645 646 if (off != s->size) 647 /* Beginning of the filler is the free pointer */ 648 print_section("Padding ", p + off, s->size - off); 649 650 dump_stack(); 651 } 652 653 void object_err(struct kmem_cache *s, struct page *page, 654 u8 *object, char *reason) 655 { 656 slab_bug(s, "%s", reason); 657 print_trailer(s, page, object); 658 } 659 660 static void slab_err(struct kmem_cache *s, struct page *page, 661 const char *fmt, ...) 662 { 663 va_list args; 664 char buf[100]; 665 666 va_start(args, fmt); 667 vsnprintf(buf, sizeof(buf), fmt, args); 668 va_end(args); 669 slab_bug(s, "%s", buf); 670 print_page_info(page); 671 dump_stack(); 672 } 673 674 static void init_object(struct kmem_cache *s, void *object, u8 val) 675 { 676 u8 *p = object; 677 678 if (s->flags & __OBJECT_POISON) { 679 memset(p, POISON_FREE, s->object_size - 1); 680 p[s->object_size - 1] = POISON_END; 681 } 682 683 if (s->flags & SLAB_RED_ZONE) 684 memset(p + s->object_size, val, s->inuse - s->object_size); 685 } 686 687 static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 688 void *from, void *to) 689 { 690 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); 691 memset(from, data, to - from); 692 } 693 694 static int check_bytes_and_report(struct kmem_cache *s, struct page *page, 695 u8 *object, char *what, 696 u8 *start, unsigned int value, unsigned int bytes) 697 { 698 u8 *fault; 699 u8 *end; 700 701 metadata_access_enable(); 702 fault = memchr_inv(start, value, bytes); 703 metadata_access_disable(); 704 if (!fault) 705 return 1; 706 707 end = start + bytes; 708 while (end > fault && end[-1] == value) 709 end--; 710 711 slab_bug(s, "%s overwritten", what); 712 pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", 713 fault, end - 1, fault[0], value); 714 print_trailer(s, page, object); 715 716 restore_bytes(s, what, value, fault, end); 717 return 0; 718 } 719 720 /* 721 * Object layout: 722 * 723 * object address 724 * Bytes of the object to be managed. 725 * If the freepointer may overlay the object then the free 726 * pointer is the first word of the object. 727 * 728 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 729 * 0xa5 (POISON_END) 730 * 731 * object + s->object_size 732 * Padding to reach word boundary. This is also used for Redzoning. 733 * Padding is extended by another word if Redzoning is enabled and 734 * object_size == inuse. 735 * 736 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with 737 * 0xcc (RED_ACTIVE) for objects in use. 738 * 739 * object + s->inuse 740 * Meta data starts here. 741 * 742 * A. Free pointer (if we cannot overwrite object on free) 743 * B. Tracking data for SLAB_STORE_USER 744 * C. Padding to reach required alignment boundary or at mininum 745 * one word if debugging is on to be able to detect writes 746 * before the word boundary. 747 * 748 * Padding is done using 0x5a (POISON_INUSE) 749 * 750 * object + s->size 751 * Nothing is used beyond s->size. 752 * 753 * If slabcaches are merged then the object_size and inuse boundaries are mostly 754 * ignored. And therefore no slab options that rely on these boundaries 755 * may be used with merged slabcaches. 756 */ 757 758 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) 759 { 760 unsigned long off = s->inuse; /* The end of info */ 761 762 if (s->offset) 763 /* Freepointer is placed after the object. */ 764 off += sizeof(void *); 765 766 if (s->flags & SLAB_STORE_USER) 767 /* We also have user information there */ 768 off += 2 * sizeof(struct track); 769 770 if (s->size == off) 771 return 1; 772 773 return check_bytes_and_report(s, page, p, "Object padding", 774 p + off, POISON_INUSE, s->size - off); 775 } 776 777 /* Check the pad bytes at the end of a slab page */ 778 static int slab_pad_check(struct kmem_cache *s, struct page *page) 779 { 780 u8 *start; 781 u8 *fault; 782 u8 *end; 783 int length; 784 int remainder; 785 786 if (!(s->flags & SLAB_POISON)) 787 return 1; 788 789 start = page_address(page); 790 length = (PAGE_SIZE << compound_order(page)) - s->reserved; 791 end = start + length; 792 remainder = length % s->size; 793 if (!remainder) 794 return 1; 795 796 metadata_access_enable(); 797 fault = memchr_inv(end - remainder, POISON_INUSE, remainder); 798 metadata_access_disable(); 799 if (!fault) 800 return 1; 801 while (end > fault && end[-1] == POISON_INUSE) 802 end--; 803 804 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); 805 print_section("Padding ", end - remainder, remainder); 806 807 restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); 808 return 0; 809 } 810 811 static int check_object(struct kmem_cache *s, struct page *page, 812 void *object, u8 val) 813 { 814 u8 *p = object; 815 u8 *endobject = object + s->object_size; 816 817 if (s->flags & SLAB_RED_ZONE) { 818 if (!check_bytes_and_report(s, page, object, "Redzone", 819 endobject, val, s->inuse - s->object_size)) 820 return 0; 821 } else { 822 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 823 check_bytes_and_report(s, page, p, "Alignment padding", 824 endobject, POISON_INUSE, 825 s->inuse - s->object_size); 826 } 827 } 828 829 if (s->flags & SLAB_POISON) { 830 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && 831 (!check_bytes_and_report(s, page, p, "Poison", p, 832 POISON_FREE, s->object_size - 1) || 833 !check_bytes_and_report(s, page, p, "Poison", 834 p + s->object_size - 1, POISON_END, 1))) 835 return 0; 836 /* 837 * check_pad_bytes cleans up on its own. 838 */ 839 check_pad_bytes(s, page, p); 840 } 841 842 if (!s->offset && val == SLUB_RED_ACTIVE) 843 /* 844 * Object and freepointer overlap. Cannot check 845 * freepointer while object is allocated. 846 */ 847 return 1; 848 849 /* Check free pointer validity */ 850 if (!check_valid_pointer(s, page, get_freepointer(s, p))) { 851 object_err(s, page, p, "Freepointer corrupt"); 852 /* 853 * No choice but to zap it and thus lose the remainder 854 * of the free objects in this slab. May cause 855 * another error because the object count is now wrong. 856 */ 857 set_freepointer(s, p, NULL); 858 return 0; 859 } 860 return 1; 861 } 862 863 static int check_slab(struct kmem_cache *s, struct page *page) 864 { 865 int maxobj; 866 867 VM_BUG_ON(!irqs_disabled()); 868 869 if (!PageSlab(page)) { 870 slab_err(s, page, "Not a valid slab page"); 871 return 0; 872 } 873 874 maxobj = order_objects(compound_order(page), s->size, s->reserved); 875 if (page->objects > maxobj) { 876 slab_err(s, page, "objects %u > max %u", 877 page->objects, maxobj); 878 return 0; 879 } 880 if (page->inuse > page->objects) { 881 slab_err(s, page, "inuse %u > max %u", 882 page->inuse, page->objects); 883 return 0; 884 } 885 /* Slab_pad_check fixes things up after itself */ 886 slab_pad_check(s, page); 887 return 1; 888 } 889 890 /* 891 * Determine if a certain object on a page is on the freelist. Must hold the 892 * slab lock to guarantee that the chains are in a consistent state. 893 */ 894 static int on_freelist(struct kmem_cache *s, struct page *page, void *search) 895 { 896 int nr = 0; 897 void *fp; 898 void *object = NULL; 899 int max_objects; 900 901 fp = page->freelist; 902 while (fp && nr <= page->objects) { 903 if (fp == search) 904 return 1; 905 if (!check_valid_pointer(s, page, fp)) { 906 if (object) { 907 object_err(s, page, object, 908 "Freechain corrupt"); 909 set_freepointer(s, object, NULL); 910 } else { 911 slab_err(s, page, "Freepointer corrupt"); 912 page->freelist = NULL; 913 page->inuse = page->objects; 914 slab_fix(s, "Freelist cleared"); 915 return 0; 916 } 917 break; 918 } 919 object = fp; 920 fp = get_freepointer(s, object); 921 nr++; 922 } 923 924 max_objects = order_objects(compound_order(page), s->size, s->reserved); 925 if (max_objects > MAX_OBJS_PER_PAGE) 926 max_objects = MAX_OBJS_PER_PAGE; 927 928 if (page->objects != max_objects) { 929 slab_err(s, page, "Wrong number of objects. Found %d but " 930 "should be %d", page->objects, max_objects); 931 page->objects = max_objects; 932 slab_fix(s, "Number of objects adjusted."); 933 } 934 if (page->inuse != page->objects - nr) { 935 slab_err(s, page, "Wrong object count. Counter is %d but " 936 "counted were %d", page->inuse, page->objects - nr); 937 page->inuse = page->objects - nr; 938 slab_fix(s, "Object count adjusted."); 939 } 940 return search == NULL; 941 } 942 943 static void trace(struct kmem_cache *s, struct page *page, void *object, 944 int alloc) 945 { 946 if (s->flags & SLAB_TRACE) { 947 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 948 s->name, 949 alloc ? "alloc" : "free", 950 object, page->inuse, 951 page->freelist); 952 953 if (!alloc) 954 print_section("Object ", (void *)object, 955 s->object_size); 956 957 dump_stack(); 958 } 959 } 960 961 /* 962 * Tracking of fully allocated slabs for debugging purposes. 963 */ 964 static void add_full(struct kmem_cache *s, 965 struct kmem_cache_node *n, struct page *page) 966 { 967 if (!(s->flags & SLAB_STORE_USER)) 968 return; 969 970 lockdep_assert_held(&n->list_lock); 971 list_add(&page->lru, &n->full); 972 } 973 974 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) 975 { 976 if (!(s->flags & SLAB_STORE_USER)) 977 return; 978 979 lockdep_assert_held(&n->list_lock); 980 list_del(&page->lru); 981 } 982 983 /* Tracking of the number of slabs for debugging purposes */ 984 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 985 { 986 struct kmem_cache_node *n = get_node(s, node); 987 988 return atomic_long_read(&n->nr_slabs); 989 } 990 991 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 992 { 993 return atomic_long_read(&n->nr_slabs); 994 } 995 996 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 997 { 998 struct kmem_cache_node *n = get_node(s, node); 999 1000 /* 1001 * May be called early in order to allocate a slab for the 1002 * kmem_cache_node structure. Solve the chicken-egg 1003 * dilemma by deferring the increment of the count during 1004 * bootstrap (see early_kmem_cache_node_alloc). 1005 */ 1006 if (likely(n)) { 1007 atomic_long_inc(&n->nr_slabs); 1008 atomic_long_add(objects, &n->total_objects); 1009 } 1010 } 1011 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 1012 { 1013 struct kmem_cache_node *n = get_node(s, node); 1014 1015 atomic_long_dec(&n->nr_slabs); 1016 atomic_long_sub(objects, &n->total_objects); 1017 } 1018 1019 /* Object debug checks for alloc/free paths */ 1020 static void setup_object_debug(struct kmem_cache *s, struct page *page, 1021 void *object) 1022 { 1023 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) 1024 return; 1025 1026 init_object(s, object, SLUB_RED_INACTIVE); 1027 init_tracking(s, object); 1028 } 1029 1030 static noinline int alloc_debug_processing(struct kmem_cache *s, 1031 struct page *page, 1032 void *object, unsigned long addr) 1033 { 1034 if (!check_slab(s, page)) 1035 goto bad; 1036 1037 if (!check_valid_pointer(s, page, object)) { 1038 object_err(s, page, object, "Freelist Pointer check fails"); 1039 goto bad; 1040 } 1041 1042 if (!check_object(s, page, object, SLUB_RED_INACTIVE)) 1043 goto bad; 1044 1045 /* Success perform special debug activities for allocs */ 1046 if (s->flags & SLAB_STORE_USER) 1047 set_track(s, object, TRACK_ALLOC, addr); 1048 trace(s, page, object, 1); 1049 init_object(s, object, SLUB_RED_ACTIVE); 1050 return 1; 1051 1052 bad: 1053 if (PageSlab(page)) { 1054 /* 1055 * If this is a slab page then lets do the best we can 1056 * to avoid issues in the future. Marking all objects 1057 * as used avoids touching the remaining objects. 1058 */ 1059 slab_fix(s, "Marking all objects used"); 1060 page->inuse = page->objects; 1061 page->freelist = NULL; 1062 } 1063 return 0; 1064 } 1065 1066 static noinline struct kmem_cache_node *free_debug_processing( 1067 struct kmem_cache *s, struct page *page, void *object, 1068 unsigned long addr, unsigned long *flags) 1069 { 1070 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1071 1072 spin_lock_irqsave(&n->list_lock, *flags); 1073 slab_lock(page); 1074 1075 if (!check_slab(s, page)) 1076 goto fail; 1077 1078 if (!check_valid_pointer(s, page, object)) { 1079 slab_err(s, page, "Invalid object pointer 0x%p", object); 1080 goto fail; 1081 } 1082 1083 if (on_freelist(s, page, object)) { 1084 object_err(s, page, object, "Object already free"); 1085 goto fail; 1086 } 1087 1088 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) 1089 goto out; 1090 1091 if (unlikely(s != page->slab_cache)) { 1092 if (!PageSlab(page)) { 1093 slab_err(s, page, "Attempt to free object(0x%p) " 1094 "outside of slab", object); 1095 } else if (!page->slab_cache) { 1096 pr_err("SLUB <none>: no slab for object 0x%p.\n", 1097 object); 1098 dump_stack(); 1099 } else 1100 object_err(s, page, object, 1101 "page slab pointer corrupt."); 1102 goto fail; 1103 } 1104 1105 if (s->flags & SLAB_STORE_USER) 1106 set_track(s, object, TRACK_FREE, addr); 1107 trace(s, page, object, 0); 1108 init_object(s, object, SLUB_RED_INACTIVE); 1109 out: 1110 slab_unlock(page); 1111 /* 1112 * Keep node_lock to preserve integrity 1113 * until the object is actually freed 1114 */ 1115 return n; 1116 1117 fail: 1118 slab_unlock(page); 1119 spin_unlock_irqrestore(&n->list_lock, *flags); 1120 slab_fix(s, "Object at 0x%p not freed", object); 1121 return NULL; 1122 } 1123 1124 static int __init setup_slub_debug(char *str) 1125 { 1126 slub_debug = DEBUG_DEFAULT_FLAGS; 1127 if (*str++ != '=' || !*str) 1128 /* 1129 * No options specified. Switch on full debugging. 1130 */ 1131 goto out; 1132 1133 if (*str == ',') 1134 /* 1135 * No options but restriction on slabs. This means full 1136 * debugging for slabs matching a pattern. 1137 */ 1138 goto check_slabs; 1139 1140 if (tolower(*str) == 'o') { 1141 /* 1142 * Avoid enabling debugging on caches if its minimum order 1143 * would increase as a result. 1144 */ 1145 disable_higher_order_debug = 1; 1146 goto out; 1147 } 1148 1149 slub_debug = 0; 1150 if (*str == '-') 1151 /* 1152 * Switch off all debugging measures. 1153 */ 1154 goto out; 1155 1156 /* 1157 * Determine which debug features should be switched on 1158 */ 1159 for (; *str && *str != ','; str++) { 1160 switch (tolower(*str)) { 1161 case 'f': 1162 slub_debug |= SLAB_DEBUG_FREE; 1163 break; 1164 case 'z': 1165 slub_debug |= SLAB_RED_ZONE; 1166 break; 1167 case 'p': 1168 slub_debug |= SLAB_POISON; 1169 break; 1170 case 'u': 1171 slub_debug |= SLAB_STORE_USER; 1172 break; 1173 case 't': 1174 slub_debug |= SLAB_TRACE; 1175 break; 1176 case 'a': 1177 slub_debug |= SLAB_FAILSLAB; 1178 break; 1179 default: 1180 pr_err("slub_debug option '%c' unknown. skipped\n", 1181 *str); 1182 } 1183 } 1184 1185 check_slabs: 1186 if (*str == ',') 1187 slub_debug_slabs = str + 1; 1188 out: 1189 return 1; 1190 } 1191 1192 __setup("slub_debug", setup_slub_debug); 1193 1194 unsigned long kmem_cache_flags(unsigned long object_size, 1195 unsigned long flags, const char *name, 1196 void (*ctor)(void *)) 1197 { 1198 /* 1199 * Enable debugging if selected on the kernel commandline. 1200 */ 1201 if (slub_debug && (!slub_debug_slabs || (name && 1202 !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) 1203 flags |= slub_debug; 1204 1205 return flags; 1206 } 1207 #else 1208 static inline void setup_object_debug(struct kmem_cache *s, 1209 struct page *page, void *object) {} 1210 1211 static inline int alloc_debug_processing(struct kmem_cache *s, 1212 struct page *page, void *object, unsigned long addr) { return 0; } 1213 1214 static inline struct kmem_cache_node *free_debug_processing( 1215 struct kmem_cache *s, struct page *page, void *object, 1216 unsigned long addr, unsigned long *flags) { return NULL; } 1217 1218 static inline int slab_pad_check(struct kmem_cache *s, struct page *page) 1219 { return 1; } 1220 static inline int check_object(struct kmem_cache *s, struct page *page, 1221 void *object, u8 val) { return 1; } 1222 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, 1223 struct page *page) {} 1224 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, 1225 struct page *page) {} 1226 unsigned long kmem_cache_flags(unsigned long object_size, 1227 unsigned long flags, const char *name, 1228 void (*ctor)(void *)) 1229 { 1230 return flags; 1231 } 1232 #define slub_debug 0 1233 1234 #define disable_higher_order_debug 0 1235 1236 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 1237 { return 0; } 1238 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1239 { return 0; } 1240 static inline void inc_slabs_node(struct kmem_cache *s, int node, 1241 int objects) {} 1242 static inline void dec_slabs_node(struct kmem_cache *s, int node, 1243 int objects) {} 1244 1245 #endif /* CONFIG_SLUB_DEBUG */ 1246 1247 /* 1248 * Hooks for other subsystems that check memory allocations. In a typical 1249 * production configuration these hooks all should produce no code at all. 1250 */ 1251 static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) 1252 { 1253 kmemleak_alloc(ptr, size, 1, flags); 1254 kasan_kmalloc_large(ptr, size); 1255 } 1256 1257 static inline void kfree_hook(const void *x) 1258 { 1259 kmemleak_free(x); 1260 kasan_kfree_large(x); 1261 } 1262 1263 static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, 1264 gfp_t flags) 1265 { 1266 flags &= gfp_allowed_mask; 1267 lockdep_trace_alloc(flags); 1268 might_sleep_if(flags & __GFP_WAIT); 1269 1270 if (should_failslab(s->object_size, flags, s->flags)) 1271 return NULL; 1272 1273 return memcg_kmem_get_cache(s, flags); 1274 } 1275 1276 static inline void slab_post_alloc_hook(struct kmem_cache *s, 1277 gfp_t flags, void *object) 1278 { 1279 flags &= gfp_allowed_mask; 1280 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); 1281 kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); 1282 memcg_kmem_put_cache(s); 1283 kasan_slab_alloc(s, object); 1284 } 1285 1286 static inline void slab_free_hook(struct kmem_cache *s, void *x) 1287 { 1288 kmemleak_free_recursive(x, s->flags); 1289 1290 /* 1291 * Trouble is that we may no longer disable interrupts in the fast path 1292 * So in order to make the debug calls that expect irqs to be 1293 * disabled we need to disable interrupts temporarily. 1294 */ 1295 #if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) 1296 { 1297 unsigned long flags; 1298 1299 local_irq_save(flags); 1300 kmemcheck_slab_free(s, x, s->object_size); 1301 debug_check_no_locks_freed(x, s->object_size); 1302 local_irq_restore(flags); 1303 } 1304 #endif 1305 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 1306 debug_check_no_obj_freed(x, s->object_size); 1307 1308 kasan_slab_free(s, x); 1309 } 1310 1311 /* 1312 * Slab allocation and freeing 1313 */ 1314 static inline struct page *alloc_slab_page(struct kmem_cache *s, 1315 gfp_t flags, int node, struct kmem_cache_order_objects oo) 1316 { 1317 struct page *page; 1318 int order = oo_order(oo); 1319 1320 flags |= __GFP_NOTRACK; 1321 1322 if (memcg_charge_slab(s, flags, order)) 1323 return NULL; 1324 1325 if (node == NUMA_NO_NODE) 1326 page = alloc_pages(flags, order); 1327 else 1328 page = alloc_pages_exact_node(node, flags, order); 1329 1330 if (!page) 1331 memcg_uncharge_slab(s, order); 1332 1333 return page; 1334 } 1335 1336 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 1337 { 1338 struct page *page; 1339 struct kmem_cache_order_objects oo = s->oo; 1340 gfp_t alloc_gfp; 1341 1342 flags &= gfp_allowed_mask; 1343 1344 if (flags & __GFP_WAIT) 1345 local_irq_enable(); 1346 1347 flags |= s->allocflags; 1348 1349 /* 1350 * Let the initial higher-order allocation fail under memory pressure 1351 * so we fall-back to the minimum order allocation. 1352 */ 1353 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 1354 1355 page = alloc_slab_page(s, alloc_gfp, node, oo); 1356 if (unlikely(!page)) { 1357 oo = s->min; 1358 alloc_gfp = flags; 1359 /* 1360 * Allocation may have failed due to fragmentation. 1361 * Try a lower order alloc if possible 1362 */ 1363 page = alloc_slab_page(s, alloc_gfp, node, oo); 1364 1365 if (page) 1366 stat(s, ORDER_FALLBACK); 1367 } 1368 1369 if (kmemcheck_enabled && page 1370 && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { 1371 int pages = 1 << oo_order(oo); 1372 1373 kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); 1374 1375 /* 1376 * Objects from caches that have a constructor don't get 1377 * cleared when they're allocated, so we need to do it here. 1378 */ 1379 if (s->ctor) 1380 kmemcheck_mark_uninitialized_pages(page, pages); 1381 else 1382 kmemcheck_mark_unallocated_pages(page, pages); 1383 } 1384 1385 if (flags & __GFP_WAIT) 1386 local_irq_disable(); 1387 if (!page) 1388 return NULL; 1389 1390 page->objects = oo_objects(oo); 1391 mod_zone_page_state(page_zone(page), 1392 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1393 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1394 1 << oo_order(oo)); 1395 1396 return page; 1397 } 1398 1399 static void setup_object(struct kmem_cache *s, struct page *page, 1400 void *object) 1401 { 1402 setup_object_debug(s, page, object); 1403 if (unlikely(s->ctor)) { 1404 kasan_unpoison_object_data(s, object); 1405 s->ctor(object); 1406 kasan_poison_object_data(s, object); 1407 } 1408 } 1409 1410 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) 1411 { 1412 struct page *page; 1413 void *start; 1414 void *p; 1415 int order; 1416 int idx; 1417 1418 if (unlikely(flags & GFP_SLAB_BUG_MASK)) { 1419 pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); 1420 BUG(); 1421 } 1422 1423 page = allocate_slab(s, 1424 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 1425 if (!page) 1426 goto out; 1427 1428 order = compound_order(page); 1429 inc_slabs_node(s, page_to_nid(page), page->objects); 1430 page->slab_cache = s; 1431 __SetPageSlab(page); 1432 if (page->pfmemalloc) 1433 SetPageSlabPfmemalloc(page); 1434 1435 start = page_address(page); 1436 1437 if (unlikely(s->flags & SLAB_POISON)) 1438 memset(start, POISON_INUSE, PAGE_SIZE << order); 1439 1440 kasan_poison_slab(page); 1441 1442 for_each_object_idx(p, idx, s, start, page->objects) { 1443 setup_object(s, page, p); 1444 if (likely(idx < page->objects)) 1445 set_freepointer(s, p, p + s->size); 1446 else 1447 set_freepointer(s, p, NULL); 1448 } 1449 1450 page->freelist = start; 1451 page->inuse = page->objects; 1452 page->frozen = 1; 1453 out: 1454 return page; 1455 } 1456 1457 static void __free_slab(struct kmem_cache *s, struct page *page) 1458 { 1459 int order = compound_order(page); 1460 int pages = 1 << order; 1461 1462 if (kmem_cache_debug(s)) { 1463 void *p; 1464 1465 slab_pad_check(s, page); 1466 for_each_object(p, s, page_address(page), 1467 page->objects) 1468 check_object(s, page, p, SLUB_RED_INACTIVE); 1469 } 1470 1471 kmemcheck_free_shadow(page, compound_order(page)); 1472 1473 mod_zone_page_state(page_zone(page), 1474 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1475 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1476 -pages); 1477 1478 __ClearPageSlabPfmemalloc(page); 1479 __ClearPageSlab(page); 1480 1481 page_mapcount_reset(page); 1482 if (current->reclaim_state) 1483 current->reclaim_state->reclaimed_slab += pages; 1484 __free_pages(page, order); 1485 memcg_uncharge_slab(s, order); 1486 } 1487 1488 #define need_reserve_slab_rcu \ 1489 (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) 1490 1491 static void rcu_free_slab(struct rcu_head *h) 1492 { 1493 struct page *page; 1494 1495 if (need_reserve_slab_rcu) 1496 page = virt_to_head_page(h); 1497 else 1498 page = container_of((struct list_head *)h, struct page, lru); 1499 1500 __free_slab(page->slab_cache, page); 1501 } 1502 1503 static void free_slab(struct kmem_cache *s, struct page *page) 1504 { 1505 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { 1506 struct rcu_head *head; 1507 1508 if (need_reserve_slab_rcu) { 1509 int order = compound_order(page); 1510 int offset = (PAGE_SIZE << order) - s->reserved; 1511 1512 VM_BUG_ON(s->reserved != sizeof(*head)); 1513 head = page_address(page) + offset; 1514 } else { 1515 /* 1516 * RCU free overloads the RCU head over the LRU 1517 */ 1518 head = (void *)&page->lru; 1519 } 1520 1521 call_rcu(head, rcu_free_slab); 1522 } else 1523 __free_slab(s, page); 1524 } 1525 1526 static void discard_slab(struct kmem_cache *s, struct page *page) 1527 { 1528 dec_slabs_node(s, page_to_nid(page), page->objects); 1529 free_slab(s, page); 1530 } 1531 1532 /* 1533 * Management of partially allocated slabs. 1534 */ 1535 static inline void 1536 __add_partial(struct kmem_cache_node *n, struct page *page, int tail) 1537 { 1538 n->nr_partial++; 1539 if (tail == DEACTIVATE_TO_TAIL) 1540 list_add_tail(&page->lru, &n->partial); 1541 else 1542 list_add(&page->lru, &n->partial); 1543 } 1544 1545 static inline void add_partial(struct kmem_cache_node *n, 1546 struct page *page, int tail) 1547 { 1548 lockdep_assert_held(&n->list_lock); 1549 __add_partial(n, page, tail); 1550 } 1551 1552 static inline void 1553 __remove_partial(struct kmem_cache_node *n, struct page *page) 1554 { 1555 list_del(&page->lru); 1556 n->nr_partial--; 1557 } 1558 1559 static inline void remove_partial(struct kmem_cache_node *n, 1560 struct page *page) 1561 { 1562 lockdep_assert_held(&n->list_lock); 1563 __remove_partial(n, page); 1564 } 1565 1566 /* 1567 * Remove slab from the partial list, freeze it and 1568 * return the pointer to the freelist. 1569 * 1570 * Returns a list of objects or NULL if it fails. 1571 */ 1572 static inline void *acquire_slab(struct kmem_cache *s, 1573 struct kmem_cache_node *n, struct page *page, 1574 int mode, int *objects) 1575 { 1576 void *freelist; 1577 unsigned long counters; 1578 struct page new; 1579 1580 lockdep_assert_held(&n->list_lock); 1581 1582 /* 1583 * Zap the freelist and set the frozen bit. 1584 * The old freelist is the list of objects for the 1585 * per cpu allocation list. 1586 */ 1587 freelist = page->freelist; 1588 counters = page->counters; 1589 new.counters = counters; 1590 *objects = new.objects - new.inuse; 1591 if (mode) { 1592 new.inuse = page->objects; 1593 new.freelist = NULL; 1594 } else { 1595 new.freelist = freelist; 1596 } 1597 1598 VM_BUG_ON(new.frozen); 1599 new.frozen = 1; 1600 1601 if (!__cmpxchg_double_slab(s, page, 1602 freelist, counters, 1603 new.freelist, new.counters, 1604 "acquire_slab")) 1605 return NULL; 1606 1607 remove_partial(n, page); 1608 WARN_ON(!freelist); 1609 return freelist; 1610 } 1611 1612 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); 1613 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); 1614 1615 /* 1616 * Try to allocate a partial slab from a specific node. 1617 */ 1618 static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, 1619 struct kmem_cache_cpu *c, gfp_t flags) 1620 { 1621 struct page *page, *page2; 1622 void *object = NULL; 1623 int available = 0; 1624 int objects; 1625 1626 /* 1627 * Racy check. If we mistakenly see no partial slabs then we 1628 * just allocate an empty slab. If we mistakenly try to get a 1629 * partial slab and there is none available then get_partials() 1630 * will return NULL. 1631 */ 1632 if (!n || !n->nr_partial) 1633 return NULL; 1634 1635 spin_lock(&n->list_lock); 1636 list_for_each_entry_safe(page, page2, &n->partial, lru) { 1637 void *t; 1638 1639 if (!pfmemalloc_match(page, flags)) 1640 continue; 1641 1642 t = acquire_slab(s, n, page, object == NULL, &objects); 1643 if (!t) 1644 break; 1645 1646 available += objects; 1647 if (!object) { 1648 c->page = page; 1649 stat(s, ALLOC_FROM_PARTIAL); 1650 object = t; 1651 } else { 1652 put_cpu_partial(s, page, 0); 1653 stat(s, CPU_PARTIAL_NODE); 1654 } 1655 if (!kmem_cache_has_cpu_partial(s) 1656 || available > s->cpu_partial / 2) 1657 break; 1658 1659 } 1660 spin_unlock(&n->list_lock); 1661 return object; 1662 } 1663 1664 /* 1665 * Get a page from somewhere. Search in increasing NUMA distances. 1666 */ 1667 static void *get_any_partial(struct kmem_cache *s, gfp_t flags, 1668 struct kmem_cache_cpu *c) 1669 { 1670 #ifdef CONFIG_NUMA 1671 struct zonelist *zonelist; 1672 struct zoneref *z; 1673 struct zone *zone; 1674 enum zone_type high_zoneidx = gfp_zone(flags); 1675 void *object; 1676 unsigned int cpuset_mems_cookie; 1677 1678 /* 1679 * The defrag ratio allows a configuration of the tradeoffs between 1680 * inter node defragmentation and node local allocations. A lower 1681 * defrag_ratio increases the tendency to do local allocations 1682 * instead of attempting to obtain partial slabs from other nodes. 1683 * 1684 * If the defrag_ratio is set to 0 then kmalloc() always 1685 * returns node local objects. If the ratio is higher then kmalloc() 1686 * may return off node objects because partial slabs are obtained 1687 * from other nodes and filled up. 1688 * 1689 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes 1690 * defrag_ratio = 1000) then every (well almost) allocation will 1691 * first attempt to defrag slab caches on other nodes. This means 1692 * scanning over all nodes to look for partial slabs which may be 1693 * expensive if we do it every time we are trying to find a slab 1694 * with available objects. 1695 */ 1696 if (!s->remote_node_defrag_ratio || 1697 get_cycles() % 1024 > s->remote_node_defrag_ratio) 1698 return NULL; 1699 1700 do { 1701 cpuset_mems_cookie = read_mems_allowed_begin(); 1702 zonelist = node_zonelist(mempolicy_slab_node(), flags); 1703 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1704 struct kmem_cache_node *n; 1705 1706 n = get_node(s, zone_to_nid(zone)); 1707 1708 if (n && cpuset_zone_allowed(zone, flags) && 1709 n->nr_partial > s->min_partial) { 1710 object = get_partial_node(s, n, c, flags); 1711 if (object) { 1712 /* 1713 * Don't check read_mems_allowed_retry() 1714 * here - if mems_allowed was updated in 1715 * parallel, that was a harmless race 1716 * between allocation and the cpuset 1717 * update 1718 */ 1719 return object; 1720 } 1721 } 1722 } 1723 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 1724 #endif 1725 return NULL; 1726 } 1727 1728 /* 1729 * Get a partial page, lock it and return it. 1730 */ 1731 static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, 1732 struct kmem_cache_cpu *c) 1733 { 1734 void *object; 1735 int searchnode = node; 1736 1737 if (node == NUMA_NO_NODE) 1738 searchnode = numa_mem_id(); 1739 else if (!node_present_pages(node)) 1740 searchnode = node_to_mem_node(node); 1741 1742 object = get_partial_node(s, get_node(s, searchnode), c, flags); 1743 if (object || node != NUMA_NO_NODE) 1744 return object; 1745 1746 return get_any_partial(s, flags, c); 1747 } 1748 1749 #ifdef CONFIG_PREEMPT 1750 /* 1751 * Calculate the next globally unique transaction for disambiguiation 1752 * during cmpxchg. The transactions start with the cpu number and are then 1753 * incremented by CONFIG_NR_CPUS. 1754 */ 1755 #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) 1756 #else 1757 /* 1758 * No preemption supported therefore also no need to check for 1759 * different cpus. 1760 */ 1761 #define TID_STEP 1 1762 #endif 1763 1764 static inline unsigned long next_tid(unsigned long tid) 1765 { 1766 return tid + TID_STEP; 1767 } 1768 1769 static inline unsigned int tid_to_cpu(unsigned long tid) 1770 { 1771 return tid % TID_STEP; 1772 } 1773 1774 static inline unsigned long tid_to_event(unsigned long tid) 1775 { 1776 return tid / TID_STEP; 1777 } 1778 1779 static inline unsigned int init_tid(int cpu) 1780 { 1781 return cpu; 1782 } 1783 1784 static inline void note_cmpxchg_failure(const char *n, 1785 const struct kmem_cache *s, unsigned long tid) 1786 { 1787 #ifdef SLUB_DEBUG_CMPXCHG 1788 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 1789 1790 pr_info("%s %s: cmpxchg redo ", n, s->name); 1791 1792 #ifdef CONFIG_PREEMPT 1793 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) 1794 pr_warn("due to cpu change %d -> %d\n", 1795 tid_to_cpu(tid), tid_to_cpu(actual_tid)); 1796 else 1797 #endif 1798 if (tid_to_event(tid) != tid_to_event(actual_tid)) 1799 pr_warn("due to cpu running other code. Event %ld->%ld\n", 1800 tid_to_event(tid), tid_to_event(actual_tid)); 1801 else 1802 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 1803 actual_tid, tid, next_tid(tid)); 1804 #endif 1805 stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 1806 } 1807 1808 static void init_kmem_cache_cpus(struct kmem_cache *s) 1809 { 1810 int cpu; 1811 1812 for_each_possible_cpu(cpu) 1813 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); 1814 } 1815 1816 /* 1817 * Remove the cpu slab 1818 */ 1819 static void deactivate_slab(struct kmem_cache *s, struct page *page, 1820 void *freelist) 1821 { 1822 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; 1823 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1824 int lock = 0; 1825 enum slab_modes l = M_NONE, m = M_NONE; 1826 void *nextfree; 1827 int tail = DEACTIVATE_TO_HEAD; 1828 struct page new; 1829 struct page old; 1830 1831 if (page->freelist) { 1832 stat(s, DEACTIVATE_REMOTE_FREES); 1833 tail = DEACTIVATE_TO_TAIL; 1834 } 1835 1836 /* 1837 * Stage one: Free all available per cpu objects back 1838 * to the page freelist while it is still frozen. Leave the 1839 * last one. 1840 * 1841 * There is no need to take the list->lock because the page 1842 * is still frozen. 1843 */ 1844 while (freelist && (nextfree = get_freepointer(s, freelist))) { 1845 void *prior; 1846 unsigned long counters; 1847 1848 do { 1849 prior = page->freelist; 1850 counters = page->counters; 1851 set_freepointer(s, freelist, prior); 1852 new.counters = counters; 1853 new.inuse--; 1854 VM_BUG_ON(!new.frozen); 1855 1856 } while (!__cmpxchg_double_slab(s, page, 1857 prior, counters, 1858 freelist, new.counters, 1859 "drain percpu freelist")); 1860 1861 freelist = nextfree; 1862 } 1863 1864 /* 1865 * Stage two: Ensure that the page is unfrozen while the 1866 * list presence reflects the actual number of objects 1867 * during unfreeze. 1868 * 1869 * We setup the list membership and then perform a cmpxchg 1870 * with the count. If there is a mismatch then the page 1871 * is not unfrozen but the page is on the wrong list. 1872 * 1873 * Then we restart the process which may have to remove 1874 * the page from the list that we just put it on again 1875 * because the number of objects in the slab may have 1876 * changed. 1877 */ 1878 redo: 1879 1880 old.freelist = page->freelist; 1881 old.counters = page->counters; 1882 VM_BUG_ON(!old.frozen); 1883 1884 /* Determine target state of the slab */ 1885 new.counters = old.counters; 1886 if (freelist) { 1887 new.inuse--; 1888 set_freepointer(s, freelist, old.freelist); 1889 new.freelist = freelist; 1890 } else 1891 new.freelist = old.freelist; 1892 1893 new.frozen = 0; 1894 1895 if (!new.inuse && n->nr_partial >= s->min_partial) 1896 m = M_FREE; 1897 else if (new.freelist) { 1898 m = M_PARTIAL; 1899 if (!lock) { 1900 lock = 1; 1901 /* 1902 * Taking the spinlock removes the possiblity 1903 * that acquire_slab() will see a slab page that 1904 * is frozen 1905 */ 1906 spin_lock(&n->list_lock); 1907 } 1908 } else { 1909 m = M_FULL; 1910 if (kmem_cache_debug(s) && !lock) { 1911 lock = 1; 1912 /* 1913 * This also ensures that the scanning of full 1914 * slabs from diagnostic functions will not see 1915 * any frozen slabs. 1916 */ 1917 spin_lock(&n->list_lock); 1918 } 1919 } 1920 1921 if (l != m) { 1922 1923 if (l == M_PARTIAL) 1924 1925 remove_partial(n, page); 1926 1927 else if (l == M_FULL) 1928 1929 remove_full(s, n, page); 1930 1931 if (m == M_PARTIAL) { 1932 1933 add_partial(n, page, tail); 1934 stat(s, tail); 1935 1936 } else if (m == M_FULL) { 1937 1938 stat(s, DEACTIVATE_FULL); 1939 add_full(s, n, page); 1940 1941 } 1942 } 1943 1944 l = m; 1945 if (!__cmpxchg_double_slab(s, page, 1946 old.freelist, old.counters, 1947 new.freelist, new.counters, 1948 "unfreezing slab")) 1949 goto redo; 1950 1951 if (lock) 1952 spin_unlock(&n->list_lock); 1953 1954 if (m == M_FREE) { 1955 stat(s, DEACTIVATE_EMPTY); 1956 discard_slab(s, page); 1957 stat(s, FREE_SLAB); 1958 } 1959 } 1960 1961 /* 1962 * Unfreeze all the cpu partial slabs. 1963 * 1964 * This function must be called with interrupts disabled 1965 * for the cpu using c (or some other guarantee must be there 1966 * to guarantee no concurrent accesses). 1967 */ 1968 static void unfreeze_partials(struct kmem_cache *s, 1969 struct kmem_cache_cpu *c) 1970 { 1971 #ifdef CONFIG_SLUB_CPU_PARTIAL 1972 struct kmem_cache_node *n = NULL, *n2 = NULL; 1973 struct page *page, *discard_page = NULL; 1974 1975 while ((page = c->partial)) { 1976 struct page new; 1977 struct page old; 1978 1979 c->partial = page->next; 1980 1981 n2 = get_node(s, page_to_nid(page)); 1982 if (n != n2) { 1983 if (n) 1984 spin_unlock(&n->list_lock); 1985 1986 n = n2; 1987 spin_lock(&n->list_lock); 1988 } 1989 1990 do { 1991 1992 old.freelist = page->freelist; 1993 old.counters = page->counters; 1994 VM_BUG_ON(!old.frozen); 1995 1996 new.counters = old.counters; 1997 new.freelist = old.freelist; 1998 1999 new.frozen = 0; 2000 2001 } while (!__cmpxchg_double_slab(s, page, 2002 old.freelist, old.counters, 2003 new.freelist, new.counters, 2004 "unfreezing slab")); 2005 2006 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { 2007 page->next = discard_page; 2008 discard_page = page; 2009 } else { 2010 add_partial(n, page, DEACTIVATE_TO_TAIL); 2011 stat(s, FREE_ADD_PARTIAL); 2012 } 2013 } 2014 2015 if (n) 2016 spin_unlock(&n->list_lock); 2017 2018 while (discard_page) { 2019 page = discard_page; 2020 discard_page = discard_page->next; 2021 2022 stat(s, DEACTIVATE_EMPTY); 2023 discard_slab(s, page); 2024 stat(s, FREE_SLAB); 2025 } 2026 #endif 2027 } 2028 2029 /* 2030 * Put a page that was just frozen (in __slab_free) into a partial page 2031 * slot if available. This is done without interrupts disabled and without 2032 * preemption disabled. The cmpxchg is racy and may put the partial page 2033 * onto a random cpus partial slot. 2034 * 2035 * If we did not find a slot then simply move all the partials to the 2036 * per node partial list. 2037 */ 2038 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) 2039 { 2040 #ifdef CONFIG_SLUB_CPU_PARTIAL 2041 struct page *oldpage; 2042 int pages; 2043 int pobjects; 2044 2045 preempt_disable(); 2046 do { 2047 pages = 0; 2048 pobjects = 0; 2049 oldpage = this_cpu_read(s->cpu_slab->partial); 2050 2051 if (oldpage) { 2052 pobjects = oldpage->pobjects; 2053 pages = oldpage->pages; 2054 if (drain && pobjects > s->cpu_partial) { 2055 unsigned long flags; 2056 /* 2057 * partial array is full. Move the existing 2058 * set to the per node partial list. 2059 */ 2060 local_irq_save(flags); 2061 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); 2062 local_irq_restore(flags); 2063 oldpage = NULL; 2064 pobjects = 0; 2065 pages = 0; 2066 stat(s, CPU_PARTIAL_DRAIN); 2067 } 2068 } 2069 2070 pages++; 2071 pobjects += page->objects - page->inuse; 2072 2073 page->pages = pages; 2074 page->pobjects = pobjects; 2075 page->next = oldpage; 2076 2077 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) 2078 != oldpage); 2079 if (unlikely(!s->cpu_partial)) { 2080 unsigned long flags; 2081 2082 local_irq_save(flags); 2083 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); 2084 local_irq_restore(flags); 2085 } 2086 preempt_enable(); 2087 #endif 2088 } 2089 2090 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 2091 { 2092 stat(s, CPUSLAB_FLUSH); 2093 deactivate_slab(s, c->page, c->freelist); 2094 2095 c->tid = next_tid(c->tid); 2096 c->page = NULL; 2097 c->freelist = NULL; 2098 } 2099 2100 /* 2101 * Flush cpu slab. 2102 * 2103 * Called from IPI handler with interrupts disabled. 2104 */ 2105 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 2106 { 2107 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 2108 2109 if (likely(c)) { 2110 if (c->page) 2111 flush_slab(s, c); 2112 2113 unfreeze_partials(s, c); 2114 } 2115 } 2116 2117 static void flush_cpu_slab(void *d) 2118 { 2119 struct kmem_cache *s = d; 2120 2121 __flush_cpu_slab(s, smp_processor_id()); 2122 } 2123 2124 static bool has_cpu_slab(int cpu, void *info) 2125 { 2126 struct kmem_cache *s = info; 2127 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 2128 2129 return c->page || c->partial; 2130 } 2131 2132 static void flush_all(struct kmem_cache *s) 2133 { 2134 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); 2135 } 2136 2137 /* 2138 * Check if the objects in a per cpu structure fit numa 2139 * locality expectations. 2140 */ 2141 static inline int node_match(struct page *page, int node) 2142 { 2143 #ifdef CONFIG_NUMA 2144 if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node)) 2145 return 0; 2146 #endif 2147 return 1; 2148 } 2149 2150 #ifdef CONFIG_SLUB_DEBUG 2151 static int count_free(struct page *page) 2152 { 2153 return page->objects - page->inuse; 2154 } 2155 2156 static inline unsigned long node_nr_objs(struct kmem_cache_node *n) 2157 { 2158 return atomic_long_read(&n->total_objects); 2159 } 2160 #endif /* CONFIG_SLUB_DEBUG */ 2161 2162 #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) 2163 static unsigned long count_partial(struct kmem_cache_node *n, 2164 int (*get_count)(struct page *)) 2165 { 2166 unsigned long flags; 2167 unsigned long x = 0; 2168 struct page *page; 2169 2170 spin_lock_irqsave(&n->list_lock, flags); 2171 list_for_each_entry(page, &n->partial, lru) 2172 x += get_count(page); 2173 spin_unlock_irqrestore(&n->list_lock, flags); 2174 return x; 2175 } 2176 #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ 2177 2178 static noinline void 2179 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) 2180 { 2181 #ifdef CONFIG_SLUB_DEBUG 2182 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 2183 DEFAULT_RATELIMIT_BURST); 2184 int node; 2185 struct kmem_cache_node *n; 2186 2187 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) 2188 return; 2189 2190 pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", 2191 nid, gfpflags); 2192 pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n", 2193 s->name, s->object_size, s->size, oo_order(s->oo), 2194 oo_order(s->min)); 2195 2196 if (oo_order(s->min) > get_order(s->object_size)) 2197 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", 2198 s->name); 2199 2200 for_each_kmem_cache_node(s, node, n) { 2201 unsigned long nr_slabs; 2202 unsigned long nr_objs; 2203 unsigned long nr_free; 2204 2205 nr_free = count_partial(n, count_free); 2206 nr_slabs = node_nr_slabs(n); 2207 nr_objs = node_nr_objs(n); 2208 2209 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", 2210 node, nr_slabs, nr_objs, nr_free); 2211 } 2212 #endif 2213 } 2214 2215 static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, 2216 int node, struct kmem_cache_cpu **pc) 2217 { 2218 void *freelist; 2219 struct kmem_cache_cpu *c = *pc; 2220 struct page *page; 2221 2222 freelist = get_partial(s, flags, node, c); 2223 2224 if (freelist) 2225 return freelist; 2226 2227 page = new_slab(s, flags, node); 2228 if (page) { 2229 c = raw_cpu_ptr(s->cpu_slab); 2230 if (c->page) 2231 flush_slab(s, c); 2232 2233 /* 2234 * No other reference to the page yet so we can 2235 * muck around with it freely without cmpxchg 2236 */ 2237 freelist = page->freelist; 2238 page->freelist = NULL; 2239 2240 stat(s, ALLOC_SLAB); 2241 c->page = page; 2242 *pc = c; 2243 } else 2244 freelist = NULL; 2245 2246 return freelist; 2247 } 2248 2249 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) 2250 { 2251 if (unlikely(PageSlabPfmemalloc(page))) 2252 return gfp_pfmemalloc_allowed(gfpflags); 2253 2254 return true; 2255 } 2256 2257 /* 2258 * Check the page->freelist of a page and either transfer the freelist to the 2259 * per cpu freelist or deactivate the page. 2260 * 2261 * The page is still frozen if the return value is not NULL. 2262 * 2263 * If this function returns NULL then the page has been unfrozen. 2264 * 2265 * This function must be called with interrupt disabled. 2266 */ 2267 static inline void *get_freelist(struct kmem_cache *s, struct page *page) 2268 { 2269 struct page new; 2270 unsigned long counters; 2271 void *freelist; 2272 2273 do { 2274 freelist = page->freelist; 2275 counters = page->counters; 2276 2277 new.counters = counters; 2278 VM_BUG_ON(!new.frozen); 2279 2280 new.inuse = page->objects; 2281 new.frozen = freelist != NULL; 2282 2283 } while (!__cmpxchg_double_slab(s, page, 2284 freelist, counters, 2285 NULL, new.counters, 2286 "get_freelist")); 2287 2288 return freelist; 2289 } 2290 2291 /* 2292 * Slow path. The lockless freelist is empty or we need to perform 2293 * debugging duties. 2294 * 2295 * Processing is still very fast if new objects have been freed to the 2296 * regular freelist. In that case we simply take over the regular freelist 2297 * as the lockless freelist and zap the regular freelist. 2298 * 2299 * If that is not working then we fall back to the partial lists. We take the 2300 * first element of the freelist as the object to allocate now and move the 2301 * rest of the freelist to the lockless freelist. 2302 * 2303 * And if we were unable to get a new slab from the partial slab lists then 2304 * we need to allocate a new slab. This is the slowest path since it involves 2305 * a call to the page allocator and the setup of a new slab. 2306 */ 2307 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 2308 unsigned long addr, struct kmem_cache_cpu *c) 2309 { 2310 void *freelist; 2311 struct page *page; 2312 unsigned long flags; 2313 2314 local_irq_save(flags); 2315 #ifdef CONFIG_PREEMPT 2316 /* 2317 * We may have been preempted and rescheduled on a different 2318 * cpu before disabling interrupts. Need to reload cpu area 2319 * pointer. 2320 */ 2321 c = this_cpu_ptr(s->cpu_slab); 2322 #endif 2323 2324 page = c->page; 2325 if (!page) 2326 goto new_slab; 2327 redo: 2328 2329 if (unlikely(!node_match(page, node))) { 2330 int searchnode = node; 2331 2332 if (node != NUMA_NO_NODE && !node_present_pages(node)) 2333 searchnode = node_to_mem_node(node); 2334 2335 if (unlikely(!node_match(page, searchnode))) { 2336 stat(s, ALLOC_NODE_MISMATCH); 2337 deactivate_slab(s, page, c->freelist); 2338 c->page = NULL; 2339 c->freelist = NULL; 2340 goto new_slab; 2341 } 2342 } 2343 2344 /* 2345 * By rights, we should be searching for a slab page that was 2346 * PFMEMALLOC but right now, we are losing the pfmemalloc 2347 * information when the page leaves the per-cpu allocator 2348 */ 2349 if (unlikely(!pfmemalloc_match(page, gfpflags))) { 2350 deactivate_slab(s, page, c->freelist); 2351 c->page = NULL; 2352 c->freelist = NULL; 2353 goto new_slab; 2354 } 2355 2356 /* must check again c->freelist in case of cpu migration or IRQ */ 2357 freelist = c->freelist; 2358 if (freelist) 2359 goto load_freelist; 2360 2361 freelist = get_freelist(s, page); 2362 2363 if (!freelist) { 2364 c->page = NULL; 2365 stat(s, DEACTIVATE_BYPASS); 2366 goto new_slab; 2367 } 2368 2369 stat(s, ALLOC_REFILL); 2370 2371 load_freelist: 2372 /* 2373 * freelist is pointing to the list of objects to be used. 2374 * page is pointing to the page from which the objects are obtained. 2375 * That page must be frozen for per cpu allocations to work. 2376 */ 2377 VM_BUG_ON(!c->page->frozen); 2378 c->freelist = get_freepointer(s, freelist); 2379 c->tid = next_tid(c->tid); 2380 local_irq_restore(flags); 2381 return freelist; 2382 2383 new_slab: 2384 2385 if (c->partial) { 2386 page = c->page = c->partial; 2387 c->partial = page->next; 2388 stat(s, CPU_PARTIAL_ALLOC); 2389 c->freelist = NULL; 2390 goto redo; 2391 } 2392 2393 freelist = new_slab_objects(s, gfpflags, node, &c); 2394 2395 if (unlikely(!freelist)) { 2396 slab_out_of_memory(s, gfpflags, node); 2397 local_irq_restore(flags); 2398 return NULL; 2399 } 2400 2401 page = c->page; 2402 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) 2403 goto load_freelist; 2404 2405 /* Only entered in the debug case */ 2406 if (kmem_cache_debug(s) && 2407 !alloc_debug_processing(s, page, freelist, addr)) 2408 goto new_slab; /* Slab failed checks. Next slab needed */ 2409 2410 deactivate_slab(s, page, get_freepointer(s, freelist)); 2411 c->page = NULL; 2412 c->freelist = NULL; 2413 local_irq_restore(flags); 2414 return freelist; 2415 } 2416 2417 /* 2418 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 2419 * have the fastpath folded into their functions. So no function call 2420 * overhead for requests that can be satisfied on the fastpath. 2421 * 2422 * The fastpath works by first checking if the lockless freelist can be used. 2423 * If not then __slab_alloc is called for slow processing. 2424 * 2425 * Otherwise we can simply pick the next object from the lockless free list. 2426 */ 2427 static __always_inline void *slab_alloc_node(struct kmem_cache *s, 2428 gfp_t gfpflags, int node, unsigned long addr) 2429 { 2430 void **object; 2431 struct kmem_cache_cpu *c; 2432 struct page *page; 2433 unsigned long tid; 2434 2435 s = slab_pre_alloc_hook(s, gfpflags); 2436 if (!s) 2437 return NULL; 2438 redo: 2439 /* 2440 * Must read kmem_cache cpu data via this cpu ptr. Preemption is 2441 * enabled. We may switch back and forth between cpus while 2442 * reading from one cpu area. That does not matter as long 2443 * as we end up on the original cpu again when doing the cmpxchg. 2444 * 2445 * We should guarantee that tid and kmem_cache are retrieved on 2446 * the same cpu. It could be different if CONFIG_PREEMPT so we need 2447 * to check if it is matched or not. 2448 */ 2449 do { 2450 tid = this_cpu_read(s->cpu_slab->tid); 2451 c = raw_cpu_ptr(s->cpu_slab); 2452 } while (IS_ENABLED(CONFIG_PREEMPT) && 2453 unlikely(tid != READ_ONCE(c->tid))); 2454 2455 /* 2456 * Irqless object alloc/free algorithm used here depends on sequence 2457 * of fetching cpu_slab's data. tid should be fetched before anything 2458 * on c to guarantee that object and page associated with previous tid 2459 * won't be used with current tid. If we fetch tid first, object and 2460 * page could be one associated with next tid and our alloc/free 2461 * request will be failed. In this case, we will retry. So, no problem. 2462 */ 2463 barrier(); 2464 2465 /* 2466 * The transaction ids are globally unique per cpu and per operation on 2467 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double 2468 * occurs on the right processor and that there was no operation on the 2469 * linked list in between. 2470 */ 2471 2472 object = c->freelist; 2473 page = c->page; 2474 if (unlikely(!object || !node_match(page, node))) { 2475 object = __slab_alloc(s, gfpflags, node, addr, c); 2476 stat(s, ALLOC_SLOWPATH); 2477 } else { 2478 void *next_object = get_freepointer_safe(s, object); 2479 2480 /* 2481 * The cmpxchg will only match if there was no additional 2482 * operation and if we are on the right processor. 2483 * 2484 * The cmpxchg does the following atomically (without lock 2485 * semantics!) 2486 * 1. Relocate first pointer to the current per cpu area. 2487 * 2. Verify that tid and freelist have not been changed 2488 * 3. If they were not changed replace tid and freelist 2489 * 2490 * Since this is without lock semantics the protection is only 2491 * against code executing on this cpu *not* from access by 2492 * other cpus. 2493 */ 2494 if (unlikely(!this_cpu_cmpxchg_double( 2495 s->cpu_slab->freelist, s->cpu_slab->tid, 2496 object, tid, 2497 next_object, next_tid(tid)))) { 2498 2499 note_cmpxchg_failure("slab_alloc", s, tid); 2500 goto redo; 2501 } 2502 prefetch_freepointer(s, next_object); 2503 stat(s, ALLOC_FASTPATH); 2504 } 2505 2506 if (unlikely(gfpflags & __GFP_ZERO) && object) 2507 memset(object, 0, s->object_size); 2508 2509 slab_post_alloc_hook(s, gfpflags, object); 2510 2511 return object; 2512 } 2513 2514 static __always_inline void *slab_alloc(struct kmem_cache *s, 2515 gfp_t gfpflags, unsigned long addr) 2516 { 2517 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); 2518 } 2519 2520 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 2521 { 2522 void *ret = slab_alloc(s, gfpflags, _RET_IP_); 2523 2524 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, 2525 s->size, gfpflags); 2526 2527 return ret; 2528 } 2529 EXPORT_SYMBOL(kmem_cache_alloc); 2530 2531 #ifdef CONFIG_TRACING 2532 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) 2533 { 2534 void *ret = slab_alloc(s, gfpflags, _RET_IP_); 2535 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); 2536 kasan_kmalloc(s, ret, size); 2537 return ret; 2538 } 2539 EXPORT_SYMBOL(kmem_cache_alloc_trace); 2540 #endif 2541 2542 #ifdef CONFIG_NUMA 2543 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 2544 { 2545 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); 2546 2547 trace_kmem_cache_alloc_node(_RET_IP_, ret, 2548 s->object_size, s->size, gfpflags, node); 2549 2550 return ret; 2551 } 2552 EXPORT_SYMBOL(kmem_cache_alloc_node); 2553 2554 #ifdef CONFIG_TRACING 2555 void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 2556 gfp_t gfpflags, 2557 int node, size_t size) 2558 { 2559 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); 2560 2561 trace_kmalloc_node(_RET_IP_, ret, 2562 size, s->size, gfpflags, node); 2563 2564 kasan_kmalloc(s, ret, size); 2565 return ret; 2566 } 2567 EXPORT_SYMBOL(kmem_cache_alloc_node_trace); 2568 #endif 2569 #endif 2570 2571 /* 2572 * Slow path handling. This may still be called frequently since objects 2573 * have a longer lifetime than the cpu slabs in most processing loads. 2574 * 2575 * So we still attempt to reduce cache line usage. Just take the slab 2576 * lock and free the item. If there is no additional partial page 2577 * handling required then we can return immediately. 2578 */ 2579 static void __slab_free(struct kmem_cache *s, struct page *page, 2580 void *x, unsigned long addr) 2581 { 2582 void *prior; 2583 void **object = (void *)x; 2584 int was_frozen; 2585 struct page new; 2586 unsigned long counters; 2587 struct kmem_cache_node *n = NULL; 2588 unsigned long uninitialized_var(flags); 2589 2590 stat(s, FREE_SLOWPATH); 2591 2592 if (kmem_cache_debug(s) && 2593 !(n = free_debug_processing(s, page, x, addr, &flags))) 2594 return; 2595 2596 do { 2597 if (unlikely(n)) { 2598 spin_unlock_irqrestore(&n->list_lock, flags); 2599 n = NULL; 2600 } 2601 prior = page->freelist; 2602 counters = page->counters; 2603 set_freepointer(s, object, prior); 2604 new.counters = counters; 2605 was_frozen = new.frozen; 2606 new.inuse--; 2607 if ((!new.inuse || !prior) && !was_frozen) { 2608 2609 if (kmem_cache_has_cpu_partial(s) && !prior) { 2610 2611 /* 2612 * Slab was on no list before and will be 2613 * partially empty 2614 * We can defer the list move and instead 2615 * freeze it. 2616 */ 2617 new.frozen = 1; 2618 2619 } else { /* Needs to be taken off a list */ 2620 2621 n = get_node(s, page_to_nid(page)); 2622 /* 2623 * Speculatively acquire the list_lock. 2624 * If the cmpxchg does not succeed then we may 2625 * drop the list_lock without any processing. 2626 * 2627 * Otherwise the list_lock will synchronize with 2628 * other processors updating the list of slabs. 2629 */ 2630 spin_lock_irqsave(&n->list_lock, flags); 2631 2632 } 2633 } 2634 2635 } while (!cmpxchg_double_slab(s, page, 2636 prior, counters, 2637 object, new.counters, 2638 "__slab_free")); 2639 2640 if (likely(!n)) { 2641 2642 /* 2643 * If we just froze the page then put it onto the 2644 * per cpu partial list. 2645 */ 2646 if (new.frozen && !was_frozen) { 2647 put_cpu_partial(s, page, 1); 2648 stat(s, CPU_PARTIAL_FREE); 2649 } 2650 /* 2651 * The list lock was not taken therefore no list 2652 * activity can be necessary. 2653 */ 2654 if (was_frozen) 2655 stat(s, FREE_FROZEN); 2656 return; 2657 } 2658 2659 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 2660 goto slab_empty; 2661 2662 /* 2663 * Objects left in the slab. If it was not on the partial list before 2664 * then add it. 2665 */ 2666 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 2667 if (kmem_cache_debug(s)) 2668 remove_full(s, n, page); 2669 add_partial(n, page, DEACTIVATE_TO_TAIL); 2670 stat(s, FREE_ADD_PARTIAL); 2671 } 2672 spin_unlock_irqrestore(&n->list_lock, flags); 2673 return; 2674 2675 slab_empty: 2676 if (prior) { 2677 /* 2678 * Slab on the partial list. 2679 */ 2680 remove_partial(n, page); 2681 stat(s, FREE_REMOVE_PARTIAL); 2682 } else { 2683 /* Slab must be on the full list */ 2684 remove_full(s, n, page); 2685 } 2686 2687 spin_unlock_irqrestore(&n->list_lock, flags); 2688 stat(s, FREE_SLAB); 2689 discard_slab(s, page); 2690 } 2691 2692 /* 2693 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 2694 * can perform fastpath freeing without additional function calls. 2695 * 2696 * The fastpath is only possible if we are freeing to the current cpu slab 2697 * of this processor. This typically the case if we have just allocated 2698 * the item before. 2699 * 2700 * If fastpath is not possible then fall back to __slab_free where we deal 2701 * with all sorts of special processing. 2702 */ 2703 static __always_inline void slab_free(struct kmem_cache *s, 2704 struct page *page, void *x, unsigned long addr) 2705 { 2706 void **object = (void *)x; 2707 struct kmem_cache_cpu *c; 2708 unsigned long tid; 2709 2710 slab_free_hook(s, x); 2711 2712 redo: 2713 /* 2714 * Determine the currently cpus per cpu slab. 2715 * The cpu may change afterward. However that does not matter since 2716 * data is retrieved via this pointer. If we are on the same cpu 2717 * during the cmpxchg then the free will succedd. 2718 */ 2719 do { 2720 tid = this_cpu_read(s->cpu_slab->tid); 2721 c = raw_cpu_ptr(s->cpu_slab); 2722 } while (IS_ENABLED(CONFIG_PREEMPT) && 2723 unlikely(tid != READ_ONCE(c->tid))); 2724 2725 /* Same with comment on barrier() in slab_alloc_node() */ 2726 barrier(); 2727 2728 if (likely(page == c->page)) { 2729 set_freepointer(s, object, c->freelist); 2730 2731 if (unlikely(!this_cpu_cmpxchg_double( 2732 s->cpu_slab->freelist, s->cpu_slab->tid, 2733 c->freelist, tid, 2734 object, next_tid(tid)))) { 2735 2736 note_cmpxchg_failure("slab_free", s, tid); 2737 goto redo; 2738 } 2739 stat(s, FREE_FASTPATH); 2740 } else 2741 __slab_free(s, page, x, addr); 2742 2743 } 2744 2745 void kmem_cache_free(struct kmem_cache *s, void *x) 2746 { 2747 s = cache_from_obj(s, x); 2748 if (!s) 2749 return; 2750 slab_free(s, virt_to_head_page(x), x, _RET_IP_); 2751 trace_kmem_cache_free(_RET_IP_, x); 2752 } 2753 EXPORT_SYMBOL(kmem_cache_free); 2754 2755 /* 2756 * Object placement in a slab is made very easy because we always start at 2757 * offset 0. If we tune the size of the object to the alignment then we can 2758 * get the required alignment by putting one properly sized object after 2759 * another. 2760 * 2761 * Notice that the allocation order determines the sizes of the per cpu 2762 * caches. Each processor has always one slab available for allocations. 2763 * Increasing the allocation order reduces the number of times that slabs 2764 * must be moved on and off the partial lists and is therefore a factor in 2765 * locking overhead. 2766 */ 2767 2768 /* 2769 * Mininum / Maximum order of slab pages. This influences locking overhead 2770 * and slab fragmentation. A higher order reduces the number of partial slabs 2771 * and increases the number of allocations possible without having to 2772 * take the list_lock. 2773 */ 2774 static int slub_min_order; 2775 static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; 2776 static int slub_min_objects; 2777 2778 /* 2779 * Calculate the order of allocation given an slab object size. 2780 * 2781 * The order of allocation has significant impact on performance and other 2782 * system components. Generally order 0 allocations should be preferred since 2783 * order 0 does not cause fragmentation in the page allocator. Larger objects 2784 * be problematic to put into order 0 slabs because there may be too much 2785 * unused space left. We go to a higher order if more than 1/16th of the slab 2786 * would be wasted. 2787 * 2788 * In order to reach satisfactory performance we must ensure that a minimum 2789 * number of objects is in one slab. Otherwise we may generate too much 2790 * activity on the partial lists which requires taking the list_lock. This is 2791 * less a concern for large slabs though which are rarely used. 2792 * 2793 * slub_max_order specifies the order where we begin to stop considering the 2794 * number of objects in a slab as critical. If we reach slub_max_order then 2795 * we try to keep the page order as low as possible. So we accept more waste 2796 * of space in favor of a small page order. 2797 * 2798 * Higher order allocations also allow the placement of more objects in a 2799 * slab and thereby reduce object handling overhead. If the user has 2800 * requested a higher mininum order then we start with that one instead of 2801 * the smallest order which will fit the object. 2802 */ 2803 static inline int slab_order(int size, int min_objects, 2804 int max_order, int fract_leftover, int reserved) 2805 { 2806 int order; 2807 int rem; 2808 int min_order = slub_min_order; 2809 2810 if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE) 2811 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 2812 2813 for (order = max(min_order, 2814 fls(min_objects * size - 1) - PAGE_SHIFT); 2815 order <= max_order; order++) { 2816 2817 unsigned long slab_size = PAGE_SIZE << order; 2818 2819 if (slab_size < min_objects * size + reserved) 2820 continue; 2821 2822 rem = (slab_size - reserved) % size; 2823 2824 if (rem <= slab_size / fract_leftover) 2825 break; 2826 2827 } 2828 2829 return order; 2830 } 2831 2832 static inline int calculate_order(int size, int reserved) 2833 { 2834 int order; 2835 int min_objects; 2836 int fraction; 2837 int max_objects; 2838 2839 /* 2840 * Attempt to find best configuration for a slab. This 2841 * works by first attempting to generate a layout with 2842 * the best configuration and backing off gradually. 2843 * 2844 * First we reduce the acceptable waste in a slab. Then 2845 * we reduce the minimum objects required in a slab. 2846 */ 2847 min_objects = slub_min_objects; 2848 if (!min_objects) 2849 min_objects = 4 * (fls(nr_cpu_ids) + 1); 2850 max_objects = order_objects(slub_max_order, size, reserved); 2851 min_objects = min(min_objects, max_objects); 2852 2853 while (min_objects > 1) { 2854 fraction = 16; 2855 while (fraction >= 4) { 2856 order = slab_order(size, min_objects, 2857 slub_max_order, fraction, reserved); 2858 if (order <= slub_max_order) 2859 return order; 2860 fraction /= 2; 2861 } 2862 min_objects--; 2863 } 2864 2865 /* 2866 * We were unable to place multiple objects in a slab. Now 2867 * lets see if we can place a single object there. 2868 */ 2869 order = slab_order(size, 1, slub_max_order, 1, reserved); 2870 if (order <= slub_max_order) 2871 return order; 2872 2873 /* 2874 * Doh this slab cannot be placed using slub_max_order. 2875 */ 2876 order = slab_order(size, 1, MAX_ORDER, 1, reserved); 2877 if (order < MAX_ORDER) 2878 return order; 2879 return -ENOSYS; 2880 } 2881 2882 static void 2883 init_kmem_cache_node(struct kmem_cache_node *n) 2884 { 2885 n->nr_partial = 0; 2886 spin_lock_init(&n->list_lock); 2887 INIT_LIST_HEAD(&n->partial); 2888 #ifdef CONFIG_SLUB_DEBUG 2889 atomic_long_set(&n->nr_slabs, 0); 2890 atomic_long_set(&n->total_objects, 0); 2891 INIT_LIST_HEAD(&n->full); 2892 #endif 2893 } 2894 2895 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 2896 { 2897 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < 2898 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu)); 2899 2900 /* 2901 * Must align to double word boundary for the double cmpxchg 2902 * instructions to work; see __pcpu_double_call_return_bool(). 2903 */ 2904 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 2905 2 * sizeof(void *)); 2906 2907 if (!s->cpu_slab) 2908 return 0; 2909 2910 init_kmem_cache_cpus(s); 2911 2912 return 1; 2913 } 2914 2915 static struct kmem_cache *kmem_cache_node; 2916 2917 /* 2918 * No kmalloc_node yet so do it by hand. We know that this is the first 2919 * slab on the node for this slabcache. There are no concurrent accesses 2920 * possible. 2921 * 2922 * Note that this function only works on the kmem_cache_node 2923 * when allocating for the kmem_cache_node. This is used for bootstrapping 2924 * memory on a fresh node that has no slab structures yet. 2925 */ 2926 static void early_kmem_cache_node_alloc(int node) 2927 { 2928 struct page *page; 2929 struct kmem_cache_node *n; 2930 2931 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); 2932 2933 page = new_slab(kmem_cache_node, GFP_NOWAIT, node); 2934 2935 BUG_ON(!page); 2936 if (page_to_nid(page) != node) { 2937 pr_err("SLUB: Unable to allocate memory from node %d\n", node); 2938 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); 2939 } 2940 2941 n = page->freelist; 2942 BUG_ON(!n); 2943 page->freelist = get_freepointer(kmem_cache_node, n); 2944 page->inuse = 1; 2945 page->frozen = 0; 2946 kmem_cache_node->node[node] = n; 2947 #ifdef CONFIG_SLUB_DEBUG 2948 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); 2949 init_tracking(kmem_cache_node, n); 2950 #endif 2951 kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); 2952 init_kmem_cache_node(n); 2953 inc_slabs_node(kmem_cache_node, node, page->objects); 2954 2955 /* 2956 * No locks need to be taken here as it has just been 2957 * initialized and there is no concurrent access. 2958 */ 2959 __add_partial(n, page, DEACTIVATE_TO_HEAD); 2960 } 2961 2962 static void free_kmem_cache_nodes(struct kmem_cache *s) 2963 { 2964 int node; 2965 struct kmem_cache_node *n; 2966 2967 for_each_kmem_cache_node(s, node, n) { 2968 kmem_cache_free(kmem_cache_node, n); 2969 s->node[node] = NULL; 2970 } 2971 } 2972 2973 static int init_kmem_cache_nodes(struct kmem_cache *s) 2974 { 2975 int node; 2976 2977 for_each_node_state(node, N_NORMAL_MEMORY) { 2978 struct kmem_cache_node *n; 2979 2980 if (slab_state == DOWN) { 2981 early_kmem_cache_node_alloc(node); 2982 continue; 2983 } 2984 n = kmem_cache_alloc_node(kmem_cache_node, 2985 GFP_KERNEL, node); 2986 2987 if (!n) { 2988 free_kmem_cache_nodes(s); 2989 return 0; 2990 } 2991 2992 s->node[node] = n; 2993 init_kmem_cache_node(n); 2994 } 2995 return 1; 2996 } 2997 2998 static void set_min_partial(struct kmem_cache *s, unsigned long min) 2999 { 3000 if (min < MIN_PARTIAL) 3001 min = MIN_PARTIAL; 3002 else if (min > MAX_PARTIAL) 3003 min = MAX_PARTIAL; 3004 s->min_partial = min; 3005 } 3006 3007 /* 3008 * calculate_sizes() determines the order and the distribution of data within 3009 * a slab object. 3010 */ 3011 static int calculate_sizes(struct kmem_cache *s, int forced_order) 3012 { 3013 unsigned long flags = s->flags; 3014 unsigned long size = s->object_size; 3015 int order; 3016 3017 /* 3018 * Round up object size to the next word boundary. We can only 3019 * place the free pointer at word boundaries and this determines 3020 * the possible location of the free pointer. 3021 */ 3022 size = ALIGN(size, sizeof(void *)); 3023 3024 #ifdef CONFIG_SLUB_DEBUG 3025 /* 3026 * Determine if we can poison the object itself. If the user of 3027 * the slab may touch the object after free or before allocation 3028 * then we should never poison the object itself. 3029 */ 3030 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && 3031 !s->ctor) 3032 s->flags |= __OBJECT_POISON; 3033 else 3034 s->flags &= ~__OBJECT_POISON; 3035 3036 3037 /* 3038 * If we are Redzoning then check if there is some space between the 3039 * end of the object and the free pointer. If not then add an 3040 * additional word to have some bytes to store Redzone information. 3041 */ 3042 if ((flags & SLAB_RED_ZONE) && size == s->object_size) 3043 size += sizeof(void *); 3044 #endif 3045 3046 /* 3047 * With that we have determined the number of bytes in actual use 3048 * by the object. This is the potential offset to the free pointer. 3049 */ 3050 s->inuse = size; 3051 3052 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || 3053 s->ctor)) { 3054 /* 3055 * Relocate free pointer after the object if it is not 3056 * permitted to overwrite the first word of the object on 3057 * kmem_cache_free. 3058 * 3059 * This is the case if we do RCU, have a constructor or 3060 * destructor or are poisoning the objects. 3061 */ 3062 s->offset = size; 3063 size += sizeof(void *); 3064 } 3065 3066 #ifdef CONFIG_SLUB_DEBUG 3067 if (flags & SLAB_STORE_USER) 3068 /* 3069 * Need to store information about allocs and frees after 3070 * the object. 3071 */ 3072 size += 2 * sizeof(struct track); 3073 3074 if (flags & SLAB_RED_ZONE) 3075 /* 3076 * Add some empty padding so that we can catch 3077 * overwrites from earlier objects rather than let 3078 * tracking information or the free pointer be 3079 * corrupted if a user writes before the start 3080 * of the object. 3081 */ 3082 size += sizeof(void *); 3083 #endif 3084 3085 /* 3086 * SLUB stores one object immediately after another beginning from 3087 * offset 0. In order to align the objects we have to simply size 3088 * each object to conform to the alignment. 3089 */ 3090 size = ALIGN(size, s->align); 3091 s->size = size; 3092 if (forced_order >= 0) 3093 order = forced_order; 3094 else 3095 order = calculate_order(size, s->reserved); 3096 3097 if (order < 0) 3098 return 0; 3099 3100 s->allocflags = 0; 3101 if (order) 3102 s->allocflags |= __GFP_COMP; 3103 3104 if (s->flags & SLAB_CACHE_DMA) 3105 s->allocflags |= GFP_DMA; 3106 3107 if (s->flags & SLAB_RECLAIM_ACCOUNT) 3108 s->allocflags |= __GFP_RECLAIMABLE; 3109 3110 /* 3111 * Determine the number of objects per slab 3112 */ 3113 s->oo = oo_make(order, size, s->reserved); 3114 s->min = oo_make(get_order(size), size, s->reserved); 3115 if (oo_objects(s->oo) > oo_objects(s->max)) 3116 s->max = s->oo; 3117 3118 return !!oo_objects(s->oo); 3119 } 3120 3121 static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) 3122 { 3123 s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); 3124 s->reserved = 0; 3125 3126 if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU)) 3127 s->reserved = sizeof(struct rcu_head); 3128 3129 if (!calculate_sizes(s, -1)) 3130 goto error; 3131 if (disable_higher_order_debug) { 3132 /* 3133 * Disable debugging flags that store metadata if the min slab 3134 * order increased. 3135 */ 3136 if (get_order(s->size) > get_order(s->object_size)) { 3137 s->flags &= ~DEBUG_METADATA_FLAGS; 3138 s->offset = 0; 3139 if (!calculate_sizes(s, -1)) 3140 goto error; 3141 } 3142 } 3143 3144 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 3145 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 3146 if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) 3147 /* Enable fast mode */ 3148 s->flags |= __CMPXCHG_DOUBLE; 3149 #endif 3150 3151 /* 3152 * The larger the object size is, the more pages we want on the partial 3153 * list to avoid pounding the page allocator excessively. 3154 */ 3155 set_min_partial(s, ilog2(s->size) / 2); 3156 3157 /* 3158 * cpu_partial determined the maximum number of objects kept in the 3159 * per cpu partial lists of a processor. 3160 * 3161 * Per cpu partial lists mainly contain slabs that just have one 3162 * object freed. If they are used for allocation then they can be 3163 * filled up again with minimal effort. The slab will never hit the 3164 * per node partial lists and therefore no locking will be required. 3165 * 3166 * This setting also determines 3167 * 3168 * A) The number of objects from per cpu partial slabs dumped to the 3169 * per node list when we reach the limit. 3170 * B) The number of objects in cpu partial slabs to extract from the 3171 * per node list when we run out of per cpu objects. We only fetch 3172 * 50% to keep some capacity around for frees. 3173 */ 3174 if (!kmem_cache_has_cpu_partial(s)) 3175 s->cpu_partial = 0; 3176 else if (s->size >= PAGE_SIZE) 3177 s->cpu_partial = 2; 3178 else if (s->size >= 1024) 3179 s->cpu_partial = 6; 3180 else if (s->size >= 256) 3181 s->cpu_partial = 13; 3182 else 3183 s->cpu_partial = 30; 3184 3185 #ifdef CONFIG_NUMA 3186 s->remote_node_defrag_ratio = 1000; 3187 #endif 3188 if (!init_kmem_cache_nodes(s)) 3189 goto error; 3190 3191 if (alloc_kmem_cache_cpus(s)) 3192 return 0; 3193 3194 free_kmem_cache_nodes(s); 3195 error: 3196 if (flags & SLAB_PANIC) 3197 panic("Cannot create slab %s size=%lu realsize=%u " 3198 "order=%u offset=%u flags=%lx\n", 3199 s->name, (unsigned long)s->size, s->size, 3200 oo_order(s->oo), s->offset, flags); 3201 return -EINVAL; 3202 } 3203 3204 static void list_slab_objects(struct kmem_cache *s, struct page *page, 3205 const char *text) 3206 { 3207 #ifdef CONFIG_SLUB_DEBUG 3208 void *addr = page_address(page); 3209 void *p; 3210 unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) * 3211 sizeof(long), GFP_ATOMIC); 3212 if (!map) 3213 return; 3214 slab_err(s, page, text, s->name); 3215 slab_lock(page); 3216 3217 get_map(s, page, map); 3218 for_each_object(p, s, addr, page->objects) { 3219 3220 if (!test_bit(slab_index(p, s, addr), map)) { 3221 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); 3222 print_tracking(s, p); 3223 } 3224 } 3225 slab_unlock(page); 3226 kfree(map); 3227 #endif 3228 } 3229 3230 /* 3231 * Attempt to free all partial slabs on a node. 3232 * This is called from kmem_cache_close(). We must be the last thread 3233 * using the cache and therefore we do not need to lock anymore. 3234 */ 3235 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 3236 { 3237 struct page *page, *h; 3238 3239 list_for_each_entry_safe(page, h, &n->partial, lru) { 3240 if (!page->inuse) { 3241 __remove_partial(n, page); 3242 discard_slab(s, page); 3243 } else { 3244 list_slab_objects(s, page, 3245 "Objects remaining in %s on kmem_cache_close()"); 3246 } 3247 } 3248 } 3249 3250 /* 3251 * Release all resources used by a slab cache. 3252 */ 3253 static inline int kmem_cache_close(struct kmem_cache *s) 3254 { 3255 int node; 3256 struct kmem_cache_node *n; 3257 3258 flush_all(s); 3259 /* Attempt to free all objects */ 3260 for_each_kmem_cache_node(s, node, n) { 3261 free_partial(s, n); 3262 if (n->nr_partial || slabs_node(s, node)) 3263 return 1; 3264 } 3265 free_percpu(s->cpu_slab); 3266 free_kmem_cache_nodes(s); 3267 return 0; 3268 } 3269 3270 int __kmem_cache_shutdown(struct kmem_cache *s) 3271 { 3272 return kmem_cache_close(s); 3273 } 3274 3275 /******************************************************************** 3276 * Kmalloc subsystem 3277 *******************************************************************/ 3278 3279 static int __init setup_slub_min_order(char *str) 3280 { 3281 get_option(&str, &slub_min_order); 3282 3283 return 1; 3284 } 3285 3286 __setup("slub_min_order=", setup_slub_min_order); 3287 3288 static int __init setup_slub_max_order(char *str) 3289 { 3290 get_option(&str, &slub_max_order); 3291 slub_max_order = min(slub_max_order, MAX_ORDER - 1); 3292 3293 return 1; 3294 } 3295 3296 __setup("slub_max_order=", setup_slub_max_order); 3297 3298 static int __init setup_slub_min_objects(char *str) 3299 { 3300 get_option(&str, &slub_min_objects); 3301 3302 return 1; 3303 } 3304 3305 __setup("slub_min_objects=", setup_slub_min_objects); 3306 3307 void *__kmalloc(size_t size, gfp_t flags) 3308 { 3309 struct kmem_cache *s; 3310 void *ret; 3311 3312 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 3313 return kmalloc_large(size, flags); 3314 3315 s = kmalloc_slab(size, flags); 3316 3317 if (unlikely(ZERO_OR_NULL_PTR(s))) 3318 return s; 3319 3320 ret = slab_alloc(s, flags, _RET_IP_); 3321 3322 trace_kmalloc(_RET_IP_, ret, size, s->size, flags); 3323 3324 kasan_kmalloc(s, ret, size); 3325 3326 return ret; 3327 } 3328 EXPORT_SYMBOL(__kmalloc); 3329 3330 #ifdef CONFIG_NUMA 3331 static void *kmalloc_large_node(size_t size, gfp_t flags, int node) 3332 { 3333 struct page *page; 3334 void *ptr = NULL; 3335 3336 flags |= __GFP_COMP | __GFP_NOTRACK; 3337 page = alloc_kmem_pages_node(node, flags, get_order(size)); 3338 if (page) 3339 ptr = page_address(page); 3340 3341 kmalloc_large_node_hook(ptr, size, flags); 3342 return ptr; 3343 } 3344 3345 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3346 { 3347 struct kmem_cache *s; 3348 void *ret; 3349 3350 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 3351 ret = kmalloc_large_node(size, flags, node); 3352 3353 trace_kmalloc_node(_RET_IP_, ret, 3354 size, PAGE_SIZE << get_order(size), 3355 flags, node); 3356 3357 return ret; 3358 } 3359 3360 s = kmalloc_slab(size, flags); 3361 3362 if (unlikely(ZERO_OR_NULL_PTR(s))) 3363 return s; 3364 3365 ret = slab_alloc_node(s, flags, node, _RET_IP_); 3366 3367 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); 3368 3369 kasan_kmalloc(s, ret, size); 3370 3371 return ret; 3372 } 3373 EXPORT_SYMBOL(__kmalloc_node); 3374 #endif 3375 3376 static size_t __ksize(const void *object) 3377 { 3378 struct page *page; 3379 3380 if (unlikely(object == ZERO_SIZE_PTR)) 3381 return 0; 3382 3383 page = virt_to_head_page(object); 3384 3385 if (unlikely(!PageSlab(page))) { 3386 WARN_ON(!PageCompound(page)); 3387 return PAGE_SIZE << compound_order(page); 3388 } 3389 3390 return slab_ksize(page->slab_cache); 3391 } 3392 3393 size_t ksize(const void *object) 3394 { 3395 size_t size = __ksize(object); 3396 /* We assume that ksize callers could use whole allocated area, 3397 so we need unpoison this area. */ 3398 kasan_krealloc(object, size); 3399 return size; 3400 } 3401 EXPORT_SYMBOL(ksize); 3402 3403 void kfree(const void *x) 3404 { 3405 struct page *page; 3406 void *object = (void *)x; 3407 3408 trace_kfree(_RET_IP_, x); 3409 3410 if (unlikely(ZERO_OR_NULL_PTR(x))) 3411 return; 3412 3413 page = virt_to_head_page(x); 3414 if (unlikely(!PageSlab(page))) { 3415 BUG_ON(!PageCompound(page)); 3416 kfree_hook(x); 3417 __free_kmem_pages(page, compound_order(page)); 3418 return; 3419 } 3420 slab_free(page->slab_cache, page, object, _RET_IP_); 3421 } 3422 EXPORT_SYMBOL(kfree); 3423 3424 #define SHRINK_PROMOTE_MAX 32 3425 3426 /* 3427 * kmem_cache_shrink discards empty slabs and promotes the slabs filled 3428 * up most to the head of the partial lists. New allocations will then 3429 * fill those up and thus they can be removed from the partial lists. 3430 * 3431 * The slabs with the least items are placed last. This results in them 3432 * being allocated from last increasing the chance that the last objects 3433 * are freed in them. 3434 */ 3435 int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate) 3436 { 3437 int node; 3438 int i; 3439 struct kmem_cache_node *n; 3440 struct page *page; 3441 struct page *t; 3442 struct list_head discard; 3443 struct list_head promote[SHRINK_PROMOTE_MAX]; 3444 unsigned long flags; 3445 int ret = 0; 3446 3447 if (deactivate) { 3448 /* 3449 * Disable empty slabs caching. Used to avoid pinning offline 3450 * memory cgroups by kmem pages that can be freed. 3451 */ 3452 s->cpu_partial = 0; 3453 s->min_partial = 0; 3454 3455 /* 3456 * s->cpu_partial is checked locklessly (see put_cpu_partial), 3457 * so we have to make sure the change is visible. 3458 */ 3459 kick_all_cpus_sync(); 3460 } 3461 3462 flush_all(s); 3463 for_each_kmem_cache_node(s, node, n) { 3464 INIT_LIST_HEAD(&discard); 3465 for (i = 0; i < SHRINK_PROMOTE_MAX; i++) 3466 INIT_LIST_HEAD(promote + i); 3467 3468 spin_lock_irqsave(&n->list_lock, flags); 3469 3470 /* 3471 * Build lists of slabs to discard or promote. 3472 * 3473 * Note that concurrent frees may occur while we hold the 3474 * list_lock. page->inuse here is the upper limit. 3475 */ 3476 list_for_each_entry_safe(page, t, &n->partial, lru) { 3477 int free = page->objects - page->inuse; 3478 3479 /* Do not reread page->inuse */ 3480 barrier(); 3481 3482 /* We do not keep full slabs on the list */ 3483 BUG_ON(free <= 0); 3484 3485 if (free == page->objects) { 3486 list_move(&page->lru, &discard); 3487 n->nr_partial--; 3488 } else if (free <= SHRINK_PROMOTE_MAX) 3489 list_move(&page->lru, promote + free - 1); 3490 } 3491 3492 /* 3493 * Promote the slabs filled up most to the head of the 3494 * partial list. 3495 */ 3496 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) 3497 list_splice(promote + i, &n->partial); 3498 3499 spin_unlock_irqrestore(&n->list_lock, flags); 3500 3501 /* Release empty slabs */ 3502 list_for_each_entry_safe(page, t, &discard, lru) 3503 discard_slab(s, page); 3504 3505 if (slabs_node(s, node)) 3506 ret = 1; 3507 } 3508 3509 return ret; 3510 } 3511 3512 static int slab_mem_going_offline_callback(void *arg) 3513 { 3514 struct kmem_cache *s; 3515 3516 mutex_lock(&slab_mutex); 3517 list_for_each_entry(s, &slab_caches, list) 3518 __kmem_cache_shrink(s, false); 3519 mutex_unlock(&slab_mutex); 3520 3521 return 0; 3522 } 3523 3524 static void slab_mem_offline_callback(void *arg) 3525 { 3526 struct kmem_cache_node *n; 3527 struct kmem_cache *s; 3528 struct memory_notify *marg = arg; 3529 int offline_node; 3530 3531 offline_node = marg->status_change_nid_normal; 3532 3533 /* 3534 * If the node still has available memory. we need kmem_cache_node 3535 * for it yet. 3536 */ 3537 if (offline_node < 0) 3538 return; 3539 3540 mutex_lock(&slab_mutex); 3541 list_for_each_entry(s, &slab_caches, list) { 3542 n = get_node(s, offline_node); 3543 if (n) { 3544 /* 3545 * if n->nr_slabs > 0, slabs still exist on the node 3546 * that is going down. We were unable to free them, 3547 * and offline_pages() function shouldn't call this 3548 * callback. So, we must fail. 3549 */ 3550 BUG_ON(slabs_node(s, offline_node)); 3551 3552 s->node[offline_node] = NULL; 3553 kmem_cache_free(kmem_cache_node, n); 3554 } 3555 } 3556 mutex_unlock(&slab_mutex); 3557 } 3558 3559 static int slab_mem_going_online_callback(void *arg) 3560 { 3561 struct kmem_cache_node *n; 3562 struct kmem_cache *s; 3563 struct memory_notify *marg = arg; 3564 int nid = marg->status_change_nid_normal; 3565 int ret = 0; 3566 3567 /* 3568 * If the node's memory is already available, then kmem_cache_node is 3569 * already created. Nothing to do. 3570 */ 3571 if (nid < 0) 3572 return 0; 3573 3574 /* 3575 * We are bringing a node online. No memory is available yet. We must 3576 * allocate a kmem_cache_node structure in order to bring the node 3577 * online. 3578 */ 3579 mutex_lock(&slab_mutex); 3580 list_for_each_entry(s, &slab_caches, list) { 3581 /* 3582 * XXX: kmem_cache_alloc_node will fallback to other nodes 3583 * since memory is not yet available from the node that 3584 * is brought up. 3585 */ 3586 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); 3587 if (!n) { 3588 ret = -ENOMEM; 3589 goto out; 3590 } 3591 init_kmem_cache_node(n); 3592 s->node[nid] = n; 3593 } 3594 out: 3595 mutex_unlock(&slab_mutex); 3596 return ret; 3597 } 3598 3599 static int slab_memory_callback(struct notifier_block *self, 3600 unsigned long action, void *arg) 3601 { 3602 int ret = 0; 3603 3604 switch (action) { 3605 case MEM_GOING_ONLINE: 3606 ret = slab_mem_going_online_callback(arg); 3607 break; 3608 case MEM_GOING_OFFLINE: 3609 ret = slab_mem_going_offline_callback(arg); 3610 break; 3611 case MEM_OFFLINE: 3612 case MEM_CANCEL_ONLINE: 3613 slab_mem_offline_callback(arg); 3614 break; 3615 case MEM_ONLINE: 3616 case MEM_CANCEL_OFFLINE: 3617 break; 3618 } 3619 if (ret) 3620 ret = notifier_from_errno(ret); 3621 else 3622 ret = NOTIFY_OK; 3623 return ret; 3624 } 3625 3626 static struct notifier_block slab_memory_callback_nb = { 3627 .notifier_call = slab_memory_callback, 3628 .priority = SLAB_CALLBACK_PRI, 3629 }; 3630 3631 /******************************************************************** 3632 * Basic setup of slabs 3633 *******************************************************************/ 3634 3635 /* 3636 * Used for early kmem_cache structures that were allocated using 3637 * the page allocator. Allocate them properly then fix up the pointers 3638 * that may be pointing to the wrong kmem_cache structure. 3639 */ 3640 3641 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 3642 { 3643 int node; 3644 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 3645 struct kmem_cache_node *n; 3646 3647 memcpy(s, static_cache, kmem_cache->object_size); 3648 3649 /* 3650 * This runs very early, and only the boot processor is supposed to be 3651 * up. Even if it weren't true, IRQs are not up so we couldn't fire 3652 * IPIs around. 3653 */ 3654 __flush_cpu_slab(s, smp_processor_id()); 3655 for_each_kmem_cache_node(s, node, n) { 3656 struct page *p; 3657 3658 list_for_each_entry(p, &n->partial, lru) 3659 p->slab_cache = s; 3660 3661 #ifdef CONFIG_SLUB_DEBUG 3662 list_for_each_entry(p, &n->full, lru) 3663 p->slab_cache = s; 3664 #endif 3665 } 3666 slab_init_memcg_params(s); 3667 list_add(&s->list, &slab_caches); 3668 return s; 3669 } 3670 3671 void __init kmem_cache_init(void) 3672 { 3673 static __initdata struct kmem_cache boot_kmem_cache, 3674 boot_kmem_cache_node; 3675 3676 if (debug_guardpage_minorder()) 3677 slub_max_order = 0; 3678 3679 kmem_cache_node = &boot_kmem_cache_node; 3680 kmem_cache = &boot_kmem_cache; 3681 3682 create_boot_cache(kmem_cache_node, "kmem_cache_node", 3683 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN); 3684 3685 register_hotmemory_notifier(&slab_memory_callback_nb); 3686 3687 /* Able to allocate the per node structures */ 3688 slab_state = PARTIAL; 3689 3690 create_boot_cache(kmem_cache, "kmem_cache", 3691 offsetof(struct kmem_cache, node) + 3692 nr_node_ids * sizeof(struct kmem_cache_node *), 3693 SLAB_HWCACHE_ALIGN); 3694 3695 kmem_cache = bootstrap(&boot_kmem_cache); 3696 3697 /* 3698 * Allocate kmem_cache_node properly from the kmem_cache slab. 3699 * kmem_cache_node is separately allocated so no need to 3700 * update any list pointers. 3701 */ 3702 kmem_cache_node = bootstrap(&boot_kmem_cache_node); 3703 3704 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 3705 create_kmalloc_caches(0); 3706 3707 #ifdef CONFIG_SMP 3708 register_cpu_notifier(&slab_notifier); 3709 #endif 3710 3711 pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n", 3712 cache_line_size(), 3713 slub_min_order, slub_max_order, slub_min_objects, 3714 nr_cpu_ids, nr_node_ids); 3715 } 3716 3717 void __init kmem_cache_init_late(void) 3718 { 3719 } 3720 3721 struct kmem_cache * 3722 __kmem_cache_alias(const char *name, size_t size, size_t align, 3723 unsigned long flags, void (*ctor)(void *)) 3724 { 3725 struct kmem_cache *s, *c; 3726 3727 s = find_mergeable(size, align, flags, name, ctor); 3728 if (s) { 3729 s->refcount++; 3730 3731 /* 3732 * Adjust the object sizes so that we clear 3733 * the complete object on kzalloc. 3734 */ 3735 s->object_size = max(s->object_size, (int)size); 3736 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); 3737 3738 for_each_memcg_cache(c, s) { 3739 c->object_size = s->object_size; 3740 c->inuse = max_t(int, c->inuse, 3741 ALIGN(size, sizeof(void *))); 3742 } 3743 3744 if (sysfs_slab_alias(s, name)) { 3745 s->refcount--; 3746 s = NULL; 3747 } 3748 } 3749 3750 return s; 3751 } 3752 3753 int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) 3754 { 3755 int err; 3756 3757 err = kmem_cache_open(s, flags); 3758 if (err) 3759 return err; 3760 3761 /* Mutex is not taken during early boot */ 3762 if (slab_state <= UP) 3763 return 0; 3764 3765 memcg_propagate_slab_attrs(s); 3766 err = sysfs_slab_add(s); 3767 if (err) 3768 kmem_cache_close(s); 3769 3770 return err; 3771 } 3772 3773 #ifdef CONFIG_SMP 3774 /* 3775 * Use the cpu notifier to insure that the cpu slabs are flushed when 3776 * necessary. 3777 */ 3778 static int slab_cpuup_callback(struct notifier_block *nfb, 3779 unsigned long action, void *hcpu) 3780 { 3781 long cpu = (long)hcpu; 3782 struct kmem_cache *s; 3783 unsigned long flags; 3784 3785 switch (action) { 3786 case CPU_UP_CANCELED: 3787 case CPU_UP_CANCELED_FROZEN: 3788 case CPU_DEAD: 3789 case CPU_DEAD_FROZEN: 3790 mutex_lock(&slab_mutex); 3791 list_for_each_entry(s, &slab_caches, list) { 3792 local_irq_save(flags); 3793 __flush_cpu_slab(s, cpu); 3794 local_irq_restore(flags); 3795 } 3796 mutex_unlock(&slab_mutex); 3797 break; 3798 default: 3799 break; 3800 } 3801 return NOTIFY_OK; 3802 } 3803 3804 static struct notifier_block slab_notifier = { 3805 .notifier_call = slab_cpuup_callback 3806 }; 3807 3808 #endif 3809 3810 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) 3811 { 3812 struct kmem_cache *s; 3813 void *ret; 3814 3815 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 3816 return kmalloc_large(size, gfpflags); 3817 3818 s = kmalloc_slab(size, gfpflags); 3819 3820 if (unlikely(ZERO_OR_NULL_PTR(s))) 3821 return s; 3822 3823 ret = slab_alloc(s, gfpflags, caller); 3824 3825 /* Honor the call site pointer we received. */ 3826 trace_kmalloc(caller, ret, size, s->size, gfpflags); 3827 3828 return ret; 3829 } 3830 3831 #ifdef CONFIG_NUMA 3832 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, 3833 int node, unsigned long caller) 3834 { 3835 struct kmem_cache *s; 3836 void *ret; 3837 3838 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 3839 ret = kmalloc_large_node(size, gfpflags, node); 3840 3841 trace_kmalloc_node(caller, ret, 3842 size, PAGE_SIZE << get_order(size), 3843 gfpflags, node); 3844 3845 return ret; 3846 } 3847 3848 s = kmalloc_slab(size, gfpflags); 3849 3850 if (unlikely(ZERO_OR_NULL_PTR(s))) 3851 return s; 3852 3853 ret = slab_alloc_node(s, gfpflags, node, caller); 3854 3855 /* Honor the call site pointer we received. */ 3856 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); 3857 3858 return ret; 3859 } 3860 #endif 3861 3862 #ifdef CONFIG_SYSFS 3863 static int count_inuse(struct page *page) 3864 { 3865 return page->inuse; 3866 } 3867 3868 static int count_total(struct page *page) 3869 { 3870 return page->objects; 3871 } 3872 #endif 3873 3874 #ifdef CONFIG_SLUB_DEBUG 3875 static int validate_slab(struct kmem_cache *s, struct page *page, 3876 unsigned long *map) 3877 { 3878 void *p; 3879 void *addr = page_address(page); 3880 3881 if (!check_slab(s, page) || 3882 !on_freelist(s, page, NULL)) 3883 return 0; 3884 3885 /* Now we know that a valid freelist exists */ 3886 bitmap_zero(map, page->objects); 3887 3888 get_map(s, page, map); 3889 for_each_object(p, s, addr, page->objects) { 3890 if (test_bit(slab_index(p, s, addr), map)) 3891 if (!check_object(s, page, p, SLUB_RED_INACTIVE)) 3892 return 0; 3893 } 3894 3895 for_each_object(p, s, addr, page->objects) 3896 if (!test_bit(slab_index(p, s, addr), map)) 3897 if (!check_object(s, page, p, SLUB_RED_ACTIVE)) 3898 return 0; 3899 return 1; 3900 } 3901 3902 static void validate_slab_slab(struct kmem_cache *s, struct page *page, 3903 unsigned long *map) 3904 { 3905 slab_lock(page); 3906 validate_slab(s, page, map); 3907 slab_unlock(page); 3908 } 3909 3910 static int validate_slab_node(struct kmem_cache *s, 3911 struct kmem_cache_node *n, unsigned long *map) 3912 { 3913 unsigned long count = 0; 3914 struct page *page; 3915 unsigned long flags; 3916 3917 spin_lock_irqsave(&n->list_lock, flags); 3918 3919 list_for_each_entry(page, &n->partial, lru) { 3920 validate_slab_slab(s, page, map); 3921 count++; 3922 } 3923 if (count != n->nr_partial) 3924 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", 3925 s->name, count, n->nr_partial); 3926 3927 if (!(s->flags & SLAB_STORE_USER)) 3928 goto out; 3929 3930 list_for_each_entry(page, &n->full, lru) { 3931 validate_slab_slab(s, page, map); 3932 count++; 3933 } 3934 if (count != atomic_long_read(&n->nr_slabs)) 3935 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", 3936 s->name, count, atomic_long_read(&n->nr_slabs)); 3937 3938 out: 3939 spin_unlock_irqrestore(&n->list_lock, flags); 3940 return count; 3941 } 3942 3943 static long validate_slab_cache(struct kmem_cache *s) 3944 { 3945 int node; 3946 unsigned long count = 0; 3947 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * 3948 sizeof(unsigned long), GFP_KERNEL); 3949 struct kmem_cache_node *n; 3950 3951 if (!map) 3952 return -ENOMEM; 3953 3954 flush_all(s); 3955 for_each_kmem_cache_node(s, node, n) 3956 count += validate_slab_node(s, n, map); 3957 kfree(map); 3958 return count; 3959 } 3960 /* 3961 * Generate lists of code addresses where slabcache objects are allocated 3962 * and freed. 3963 */ 3964 3965 struct location { 3966 unsigned long count; 3967 unsigned long addr; 3968 long long sum_time; 3969 long min_time; 3970 long max_time; 3971 long min_pid; 3972 long max_pid; 3973 DECLARE_BITMAP(cpus, NR_CPUS); 3974 nodemask_t nodes; 3975 }; 3976 3977 struct loc_track { 3978 unsigned long max; 3979 unsigned long count; 3980 struct location *loc; 3981 }; 3982 3983 static void free_loc_track(struct loc_track *t) 3984 { 3985 if (t->max) 3986 free_pages((unsigned long)t->loc, 3987 get_order(sizeof(struct location) * t->max)); 3988 } 3989 3990 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 3991 { 3992 struct location *l; 3993 int order; 3994 3995 order = get_order(sizeof(struct location) * max); 3996 3997 l = (void *)__get_free_pages(flags, order); 3998 if (!l) 3999 return 0; 4000 4001 if (t->count) { 4002 memcpy(l, t->loc, sizeof(struct location) * t->count); 4003 free_loc_track(t); 4004 } 4005 t->max = max; 4006 t->loc = l; 4007 return 1; 4008 } 4009 4010 static int add_location(struct loc_track *t, struct kmem_cache *s, 4011 const struct track *track) 4012 { 4013 long start, end, pos; 4014 struct location *l; 4015 unsigned long caddr; 4016 unsigned long age = jiffies - track->when; 4017 4018 start = -1; 4019 end = t->count; 4020 4021 for ( ; ; ) { 4022 pos = start + (end - start + 1) / 2; 4023 4024 /* 4025 * There is nothing at "end". If we end up there 4026 * we need to add something to before end. 4027 */ 4028 if (pos == end) 4029 break; 4030 4031 caddr = t->loc[pos].addr; 4032 if (track->addr == caddr) { 4033 4034 l = &t->loc[pos]; 4035 l->count++; 4036 if (track->when) { 4037 l->sum_time += age; 4038 if (age < l->min_time) 4039 l->min_time = age; 4040 if (age > l->max_time) 4041 l->max_time = age; 4042 4043 if (track->pid < l->min_pid) 4044 l->min_pid = track->pid; 4045 if (track->pid > l->max_pid) 4046 l->max_pid = track->pid; 4047 4048 cpumask_set_cpu(track->cpu, 4049 to_cpumask(l->cpus)); 4050 } 4051 node_set(page_to_nid(virt_to_page(track)), l->nodes); 4052 return 1; 4053 } 4054 4055 if (track->addr < caddr) 4056 end = pos; 4057 else 4058 start = pos; 4059 } 4060 4061 /* 4062 * Not found. Insert new tracking element. 4063 */ 4064 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 4065 return 0; 4066 4067 l = t->loc + pos; 4068 if (pos < t->count) 4069 memmove(l + 1, l, 4070 (t->count - pos) * sizeof(struct location)); 4071 t->count++; 4072 l->count = 1; 4073 l->addr = track->addr; 4074 l->sum_time = age; 4075 l->min_time = age; 4076 l->max_time = age; 4077 l->min_pid = track->pid; 4078 l->max_pid = track->pid; 4079 cpumask_clear(to_cpumask(l->cpus)); 4080 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 4081 nodes_clear(l->nodes); 4082 node_set(page_to_nid(virt_to_page(track)), l->nodes); 4083 return 1; 4084 } 4085 4086 static void process_slab(struct loc_track *t, struct kmem_cache *s, 4087 struct page *page, enum track_item alloc, 4088 unsigned long *map) 4089 { 4090 void *addr = page_address(page); 4091 void *p; 4092 4093 bitmap_zero(map, page->objects); 4094 get_map(s, page, map); 4095 4096 for_each_object(p, s, addr, page->objects) 4097 if (!test_bit(slab_index(p, s, addr), map)) 4098 add_location(t, s, get_track(s, p, alloc)); 4099 } 4100 4101 static int list_locations(struct kmem_cache *s, char *buf, 4102 enum track_item alloc) 4103 { 4104 int len = 0; 4105 unsigned long i; 4106 struct loc_track t = { 0, 0, NULL }; 4107 int node; 4108 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * 4109 sizeof(unsigned long), GFP_KERNEL); 4110 struct kmem_cache_node *n; 4111 4112 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), 4113 GFP_TEMPORARY)) { 4114 kfree(map); 4115 return sprintf(buf, "Out of memory\n"); 4116 } 4117 /* Push back cpu slabs */ 4118 flush_all(s); 4119 4120 for_each_kmem_cache_node(s, node, n) { 4121 unsigned long flags; 4122 struct page *page; 4123 4124 if (!atomic_long_read(&n->nr_slabs)) 4125 continue; 4126 4127 spin_lock_irqsave(&n->list_lock, flags); 4128 list_for_each_entry(page, &n->partial, lru) 4129 process_slab(&t, s, page, alloc, map); 4130 list_for_each_entry(page, &n->full, lru) 4131 process_slab(&t, s, page, alloc, map); 4132 spin_unlock_irqrestore(&n->list_lock, flags); 4133 } 4134 4135 for (i = 0; i < t.count; i++) { 4136 struct location *l = &t.loc[i]; 4137 4138 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) 4139 break; 4140 len += sprintf(buf + len, "%7ld ", l->count); 4141 4142 if (l->addr) 4143 len += sprintf(buf + len, "%pS", (void *)l->addr); 4144 else 4145 len += sprintf(buf + len, "<not-available>"); 4146 4147 if (l->sum_time != l->min_time) { 4148 len += sprintf(buf + len, " age=%ld/%ld/%ld", 4149 l->min_time, 4150 (long)div_u64(l->sum_time, l->count), 4151 l->max_time); 4152 } else 4153 len += sprintf(buf + len, " age=%ld", 4154 l->min_time); 4155 4156 if (l->min_pid != l->max_pid) 4157 len += sprintf(buf + len, " pid=%ld-%ld", 4158 l->min_pid, l->max_pid); 4159 else 4160 len += sprintf(buf + len, " pid=%ld", 4161 l->min_pid); 4162 4163 if (num_online_cpus() > 1 && 4164 !cpumask_empty(to_cpumask(l->cpus)) && 4165 len < PAGE_SIZE - 60) 4166 len += scnprintf(buf + len, PAGE_SIZE - len - 50, 4167 " cpus=%*pbl", 4168 cpumask_pr_args(to_cpumask(l->cpus))); 4169 4170 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && 4171 len < PAGE_SIZE - 60) 4172 len += scnprintf(buf + len, PAGE_SIZE - len - 50, 4173 " nodes=%*pbl", 4174 nodemask_pr_args(&l->nodes)); 4175 4176 len += sprintf(buf + len, "\n"); 4177 } 4178 4179 free_loc_track(&t); 4180 kfree(map); 4181 if (!t.count) 4182 len += sprintf(buf, "No data\n"); 4183 return len; 4184 } 4185 #endif 4186 4187 #ifdef SLUB_RESILIENCY_TEST 4188 static void __init resiliency_test(void) 4189 { 4190 u8 *p; 4191 4192 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10); 4193 4194 pr_err("SLUB resiliency testing\n"); 4195 pr_err("-----------------------\n"); 4196 pr_err("A. Corruption after allocation\n"); 4197 4198 p = kzalloc(16, GFP_KERNEL); 4199 p[16] = 0x12; 4200 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", 4201 p + 16); 4202 4203 validate_slab_cache(kmalloc_caches[4]); 4204 4205 /* Hmmm... The next two are dangerous */ 4206 p = kzalloc(32, GFP_KERNEL); 4207 p[32 + sizeof(void *)] = 0x34; 4208 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", 4209 p); 4210 pr_err("If allocated object is overwritten then not detectable\n\n"); 4211 4212 validate_slab_cache(kmalloc_caches[5]); 4213 p = kzalloc(64, GFP_KERNEL); 4214 p += 64 + (get_cycles() & 0xff) * sizeof(void *); 4215 *p = 0x56; 4216 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", 4217 p); 4218 pr_err("If allocated object is overwritten then not detectable\n\n"); 4219 validate_slab_cache(kmalloc_caches[6]); 4220 4221 pr_err("\nB. Corruption after free\n"); 4222 p = kzalloc(128, GFP_KERNEL); 4223 kfree(p); 4224 *p = 0x78; 4225 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); 4226 validate_slab_cache(kmalloc_caches[7]); 4227 4228 p = kzalloc(256, GFP_KERNEL); 4229 kfree(p); 4230 p[50] = 0x9a; 4231 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); 4232 validate_slab_cache(kmalloc_caches[8]); 4233 4234 p = kzalloc(512, GFP_KERNEL); 4235 kfree(p); 4236 p[512] = 0xab; 4237 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); 4238 validate_slab_cache(kmalloc_caches[9]); 4239 } 4240 #else 4241 #ifdef CONFIG_SYSFS 4242 static void resiliency_test(void) {}; 4243 #endif 4244 #endif 4245 4246 #ifdef CONFIG_SYSFS 4247 enum slab_stat_type { 4248 SL_ALL, /* All slabs */ 4249 SL_PARTIAL, /* Only partially allocated slabs */ 4250 SL_CPU, /* Only slabs used for cpu caches */ 4251 SL_OBJECTS, /* Determine allocated objects not slabs */ 4252 SL_TOTAL /* Determine object capacity not slabs */ 4253 }; 4254 4255 #define SO_ALL (1 << SL_ALL) 4256 #define SO_PARTIAL (1 << SL_PARTIAL) 4257 #define SO_CPU (1 << SL_CPU) 4258 #define SO_OBJECTS (1 << SL_OBJECTS) 4259 #define SO_TOTAL (1 << SL_TOTAL) 4260 4261 static ssize_t show_slab_objects(struct kmem_cache *s, 4262 char *buf, unsigned long flags) 4263 { 4264 unsigned long total = 0; 4265 int node; 4266 int x; 4267 unsigned long *nodes; 4268 4269 nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); 4270 if (!nodes) 4271 return -ENOMEM; 4272 4273 if (flags & SO_CPU) { 4274 int cpu; 4275 4276 for_each_possible_cpu(cpu) { 4277 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 4278 cpu); 4279 int node; 4280 struct page *page; 4281 4282 page = ACCESS_ONCE(c->page); 4283 if (!page) 4284 continue; 4285 4286 node = page_to_nid(page); 4287 if (flags & SO_TOTAL) 4288 x = page->objects; 4289 else if (flags & SO_OBJECTS) 4290 x = page->inuse; 4291 else 4292 x = 1; 4293 4294 total += x; 4295 nodes[node] += x; 4296 4297 page = ACCESS_ONCE(c->partial); 4298 if (page) { 4299 node = page_to_nid(page); 4300 if (flags & SO_TOTAL) 4301 WARN_ON_ONCE(1); 4302 else if (flags & SO_OBJECTS) 4303 WARN_ON_ONCE(1); 4304 else 4305 x = page->pages; 4306 total += x; 4307 nodes[node] += x; 4308 } 4309 } 4310 } 4311 4312 get_online_mems(); 4313 #ifdef CONFIG_SLUB_DEBUG 4314 if (flags & SO_ALL) { 4315 struct kmem_cache_node *n; 4316 4317 for_each_kmem_cache_node(s, node, n) { 4318 4319 if (flags & SO_TOTAL) 4320 x = atomic_long_read(&n->total_objects); 4321 else if (flags & SO_OBJECTS) 4322 x = atomic_long_read(&n->total_objects) - 4323 count_partial(n, count_free); 4324 else 4325 x = atomic_long_read(&n->nr_slabs); 4326 total += x; 4327 nodes[node] += x; 4328 } 4329 4330 } else 4331 #endif 4332 if (flags & SO_PARTIAL) { 4333 struct kmem_cache_node *n; 4334 4335 for_each_kmem_cache_node(s, node, n) { 4336 if (flags & SO_TOTAL) 4337 x = count_partial(n, count_total); 4338 else if (flags & SO_OBJECTS) 4339 x = count_partial(n, count_inuse); 4340 else 4341 x = n->nr_partial; 4342 total += x; 4343 nodes[node] += x; 4344 } 4345 } 4346 x = sprintf(buf, "%lu", total); 4347 #ifdef CONFIG_NUMA 4348 for (node = 0; node < nr_node_ids; node++) 4349 if (nodes[node]) 4350 x += sprintf(buf + x, " N%d=%lu", 4351 node, nodes[node]); 4352 #endif 4353 put_online_mems(); 4354 kfree(nodes); 4355 return x + sprintf(buf + x, "\n"); 4356 } 4357 4358 #ifdef CONFIG_SLUB_DEBUG 4359 static int any_slab_objects(struct kmem_cache *s) 4360 { 4361 int node; 4362 struct kmem_cache_node *n; 4363 4364 for_each_kmem_cache_node(s, node, n) 4365 if (atomic_long_read(&n->total_objects)) 4366 return 1; 4367 4368 return 0; 4369 } 4370 #endif 4371 4372 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 4373 #define to_slab(n) container_of(n, struct kmem_cache, kobj) 4374 4375 struct slab_attribute { 4376 struct attribute attr; 4377 ssize_t (*show)(struct kmem_cache *s, char *buf); 4378 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 4379 }; 4380 4381 #define SLAB_ATTR_RO(_name) \ 4382 static struct slab_attribute _name##_attr = \ 4383 __ATTR(_name, 0400, _name##_show, NULL) 4384 4385 #define SLAB_ATTR(_name) \ 4386 static struct slab_attribute _name##_attr = \ 4387 __ATTR(_name, 0600, _name##_show, _name##_store) 4388 4389 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 4390 { 4391 return sprintf(buf, "%d\n", s->size); 4392 } 4393 SLAB_ATTR_RO(slab_size); 4394 4395 static ssize_t align_show(struct kmem_cache *s, char *buf) 4396 { 4397 return sprintf(buf, "%d\n", s->align); 4398 } 4399 SLAB_ATTR_RO(align); 4400 4401 static ssize_t object_size_show(struct kmem_cache *s, char *buf) 4402 { 4403 return sprintf(buf, "%d\n", s->object_size); 4404 } 4405 SLAB_ATTR_RO(object_size); 4406 4407 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 4408 { 4409 return sprintf(buf, "%d\n", oo_objects(s->oo)); 4410 } 4411 SLAB_ATTR_RO(objs_per_slab); 4412 4413 static ssize_t order_store(struct kmem_cache *s, 4414 const char *buf, size_t length) 4415 { 4416 unsigned long order; 4417 int err; 4418 4419 err = kstrtoul(buf, 10, &order); 4420 if (err) 4421 return err; 4422 4423 if (order > slub_max_order || order < slub_min_order) 4424 return -EINVAL; 4425 4426 calculate_sizes(s, order); 4427 return length; 4428 } 4429 4430 static ssize_t order_show(struct kmem_cache *s, char *buf) 4431 { 4432 return sprintf(buf, "%d\n", oo_order(s->oo)); 4433 } 4434 SLAB_ATTR(order); 4435 4436 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 4437 { 4438 return sprintf(buf, "%lu\n", s->min_partial); 4439 } 4440 4441 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 4442 size_t length) 4443 { 4444 unsigned long min; 4445 int err; 4446 4447 err = kstrtoul(buf, 10, &min); 4448 if (err) 4449 return err; 4450 4451 set_min_partial(s, min); 4452 return length; 4453 } 4454 SLAB_ATTR(min_partial); 4455 4456 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) 4457 { 4458 return sprintf(buf, "%u\n", s->cpu_partial); 4459 } 4460 4461 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, 4462 size_t length) 4463 { 4464 unsigned long objects; 4465 int err; 4466 4467 err = kstrtoul(buf, 10, &objects); 4468 if (err) 4469 return err; 4470 if (objects && !kmem_cache_has_cpu_partial(s)) 4471 return -EINVAL; 4472 4473 s->cpu_partial = objects; 4474 flush_all(s); 4475 return length; 4476 } 4477 SLAB_ATTR(cpu_partial); 4478 4479 static ssize_t ctor_show(struct kmem_cache *s, char *buf) 4480 { 4481 if (!s->ctor) 4482 return 0; 4483 return sprintf(buf, "%pS\n", s->ctor); 4484 } 4485 SLAB_ATTR_RO(ctor); 4486 4487 static ssize_t aliases_show(struct kmem_cache *s, char *buf) 4488 { 4489 return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); 4490 } 4491 SLAB_ATTR_RO(aliases); 4492 4493 static ssize_t partial_show(struct kmem_cache *s, char *buf) 4494 { 4495 return show_slab_objects(s, buf, SO_PARTIAL); 4496 } 4497 SLAB_ATTR_RO(partial); 4498 4499 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 4500 { 4501 return show_slab_objects(s, buf, SO_CPU); 4502 } 4503 SLAB_ATTR_RO(cpu_slabs); 4504 4505 static ssize_t objects_show(struct kmem_cache *s, char *buf) 4506 { 4507 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 4508 } 4509 SLAB_ATTR_RO(objects); 4510 4511 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 4512 { 4513 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 4514 } 4515 SLAB_ATTR_RO(objects_partial); 4516 4517 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) 4518 { 4519 int objects = 0; 4520 int pages = 0; 4521 int cpu; 4522 int len; 4523 4524 for_each_online_cpu(cpu) { 4525 struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; 4526 4527 if (page) { 4528 pages += page->pages; 4529 objects += page->pobjects; 4530 } 4531 } 4532 4533 len = sprintf(buf, "%d(%d)", objects, pages); 4534 4535 #ifdef CONFIG_SMP 4536 for_each_online_cpu(cpu) { 4537 struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; 4538 4539 if (page && len < PAGE_SIZE - 20) 4540 len += sprintf(buf + len, " C%d=%d(%d)", cpu, 4541 page->pobjects, page->pages); 4542 } 4543 #endif 4544 return len + sprintf(buf + len, "\n"); 4545 } 4546 SLAB_ATTR_RO(slabs_cpu_partial); 4547 4548 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 4549 { 4550 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 4551 } 4552 4553 static ssize_t reclaim_account_store(struct kmem_cache *s, 4554 const char *buf, size_t length) 4555 { 4556 s->flags &= ~SLAB_RECLAIM_ACCOUNT; 4557 if (buf[0] == '1') 4558 s->flags |= SLAB_RECLAIM_ACCOUNT; 4559 return length; 4560 } 4561 SLAB_ATTR(reclaim_account); 4562 4563 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 4564 { 4565 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 4566 } 4567 SLAB_ATTR_RO(hwcache_align); 4568 4569 #ifdef CONFIG_ZONE_DMA 4570 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 4571 { 4572 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 4573 } 4574 SLAB_ATTR_RO(cache_dma); 4575 #endif 4576 4577 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 4578 { 4579 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); 4580 } 4581 SLAB_ATTR_RO(destroy_by_rcu); 4582 4583 static ssize_t reserved_show(struct kmem_cache *s, char *buf) 4584 { 4585 return sprintf(buf, "%d\n", s->reserved); 4586 } 4587 SLAB_ATTR_RO(reserved); 4588 4589 #ifdef CONFIG_SLUB_DEBUG 4590 static ssize_t slabs_show(struct kmem_cache *s, char *buf) 4591 { 4592 return show_slab_objects(s, buf, SO_ALL); 4593 } 4594 SLAB_ATTR_RO(slabs); 4595 4596 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 4597 { 4598 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 4599 } 4600 SLAB_ATTR_RO(total_objects); 4601 4602 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 4603 { 4604 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); 4605 } 4606 4607 static ssize_t sanity_checks_store(struct kmem_cache *s, 4608 const char *buf, size_t length) 4609 { 4610 s->flags &= ~SLAB_DEBUG_FREE; 4611 if (buf[0] == '1') { 4612 s->flags &= ~__CMPXCHG_DOUBLE; 4613 s->flags |= SLAB_DEBUG_FREE; 4614 } 4615 return length; 4616 } 4617 SLAB_ATTR(sanity_checks); 4618 4619 static ssize_t trace_show(struct kmem_cache *s, char *buf) 4620 { 4621 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 4622 } 4623 4624 static ssize_t trace_store(struct kmem_cache *s, const char *buf, 4625 size_t length) 4626 { 4627 /* 4628 * Tracing a merged cache is going to give confusing results 4629 * as well as cause other issues like converting a mergeable 4630 * cache into an umergeable one. 4631 */ 4632 if (s->refcount > 1) 4633 return -EINVAL; 4634 4635 s->flags &= ~SLAB_TRACE; 4636 if (buf[0] == '1') { 4637 s->flags &= ~__CMPXCHG_DOUBLE; 4638 s->flags |= SLAB_TRACE; 4639 } 4640 return length; 4641 } 4642 SLAB_ATTR(trace); 4643 4644 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 4645 { 4646 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 4647 } 4648 4649 static ssize_t red_zone_store(struct kmem_cache *s, 4650 const char *buf, size_t length) 4651 { 4652 if (any_slab_objects(s)) 4653 return -EBUSY; 4654 4655 s->flags &= ~SLAB_RED_ZONE; 4656 if (buf[0] == '1') { 4657 s->flags &= ~__CMPXCHG_DOUBLE; 4658 s->flags |= SLAB_RED_ZONE; 4659 } 4660 calculate_sizes(s, -1); 4661 return length; 4662 } 4663 SLAB_ATTR(red_zone); 4664 4665 static ssize_t poison_show(struct kmem_cache *s, char *buf) 4666 { 4667 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); 4668 } 4669 4670 static ssize_t poison_store(struct kmem_cache *s, 4671 const char *buf, size_t length) 4672 { 4673 if (any_slab_objects(s)) 4674 return -EBUSY; 4675 4676 s->flags &= ~SLAB_POISON; 4677 if (buf[0] == '1') { 4678 s->flags &= ~__CMPXCHG_DOUBLE; 4679 s->flags |= SLAB_POISON; 4680 } 4681 calculate_sizes(s, -1); 4682 return length; 4683 } 4684 SLAB_ATTR(poison); 4685 4686 static ssize_t store_user_show(struct kmem_cache *s, char *buf) 4687 { 4688 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 4689 } 4690 4691 static ssize_t store_user_store(struct kmem_cache *s, 4692 const char *buf, size_t length) 4693 { 4694 if (any_slab_objects(s)) 4695 return -EBUSY; 4696 4697 s->flags &= ~SLAB_STORE_USER; 4698 if (buf[0] == '1') { 4699 s->flags &= ~__CMPXCHG_DOUBLE; 4700 s->flags |= SLAB_STORE_USER; 4701 } 4702 calculate_sizes(s, -1); 4703 return length; 4704 } 4705 SLAB_ATTR(store_user); 4706 4707 static ssize_t validate_show(struct kmem_cache *s, char *buf) 4708 { 4709 return 0; 4710 } 4711 4712 static ssize_t validate_store(struct kmem_cache *s, 4713 const char *buf, size_t length) 4714 { 4715 int ret = -EINVAL; 4716 4717 if (buf[0] == '1') { 4718 ret = validate_slab_cache(s); 4719 if (ret >= 0) 4720 ret = length; 4721 } 4722 return ret; 4723 } 4724 SLAB_ATTR(validate); 4725 4726 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) 4727 { 4728 if (!(s->flags & SLAB_STORE_USER)) 4729 return -ENOSYS; 4730 return list_locations(s, buf, TRACK_ALLOC); 4731 } 4732 SLAB_ATTR_RO(alloc_calls); 4733 4734 static ssize_t free_calls_show(struct kmem_cache *s, char *buf) 4735 { 4736 if (!(s->flags & SLAB_STORE_USER)) 4737 return -ENOSYS; 4738 return list_locations(s, buf, TRACK_FREE); 4739 } 4740 SLAB_ATTR_RO(free_calls); 4741 #endif /* CONFIG_SLUB_DEBUG */ 4742 4743 #ifdef CONFIG_FAILSLAB 4744 static ssize_t failslab_show(struct kmem_cache *s, char *buf) 4745 { 4746 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); 4747 } 4748 4749 static ssize_t failslab_store(struct kmem_cache *s, const char *buf, 4750 size_t length) 4751 { 4752 if (s->refcount > 1) 4753 return -EINVAL; 4754 4755 s->flags &= ~SLAB_FAILSLAB; 4756 if (buf[0] == '1') 4757 s->flags |= SLAB_FAILSLAB; 4758 return length; 4759 } 4760 SLAB_ATTR(failslab); 4761 #endif 4762 4763 static ssize_t shrink_show(struct kmem_cache *s, char *buf) 4764 { 4765 return 0; 4766 } 4767 4768 static ssize_t shrink_store(struct kmem_cache *s, 4769 const char *buf, size_t length) 4770 { 4771 if (buf[0] == '1') 4772 kmem_cache_shrink(s); 4773 else 4774 return -EINVAL; 4775 return length; 4776 } 4777 SLAB_ATTR(shrink); 4778 4779 #ifdef CONFIG_NUMA 4780 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 4781 { 4782 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); 4783 } 4784 4785 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 4786 const char *buf, size_t length) 4787 { 4788 unsigned long ratio; 4789 int err; 4790 4791 err = kstrtoul(buf, 10, &ratio); 4792 if (err) 4793 return err; 4794 4795 if (ratio <= 100) 4796 s->remote_node_defrag_ratio = ratio * 10; 4797 4798 return length; 4799 } 4800 SLAB_ATTR(remote_node_defrag_ratio); 4801 #endif 4802 4803 #ifdef CONFIG_SLUB_STATS 4804 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 4805 { 4806 unsigned long sum = 0; 4807 int cpu; 4808 int len; 4809 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); 4810 4811 if (!data) 4812 return -ENOMEM; 4813 4814 for_each_online_cpu(cpu) { 4815 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; 4816 4817 data[cpu] = x; 4818 sum += x; 4819 } 4820 4821 len = sprintf(buf, "%lu", sum); 4822 4823 #ifdef CONFIG_SMP 4824 for_each_online_cpu(cpu) { 4825 if (data[cpu] && len < PAGE_SIZE - 20) 4826 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); 4827 } 4828 #endif 4829 kfree(data); 4830 return len + sprintf(buf + len, "\n"); 4831 } 4832 4833 static void clear_stat(struct kmem_cache *s, enum stat_item si) 4834 { 4835 int cpu; 4836 4837 for_each_online_cpu(cpu) 4838 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; 4839 } 4840 4841 #define STAT_ATTR(si, text) \ 4842 static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 4843 { \ 4844 return show_stat(s, buf, si); \ 4845 } \ 4846 static ssize_t text##_store(struct kmem_cache *s, \ 4847 const char *buf, size_t length) \ 4848 { \ 4849 if (buf[0] != '0') \ 4850 return -EINVAL; \ 4851 clear_stat(s, si); \ 4852 return length; \ 4853 } \ 4854 SLAB_ATTR(text); \ 4855 4856 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 4857 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 4858 STAT_ATTR(FREE_FASTPATH, free_fastpath); 4859 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 4860 STAT_ATTR(FREE_FROZEN, free_frozen); 4861 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 4862 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 4863 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 4864 STAT_ATTR(ALLOC_SLAB, alloc_slab); 4865 STAT_ATTR(ALLOC_REFILL, alloc_refill); 4866 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 4867 STAT_ATTR(FREE_SLAB, free_slab); 4868 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 4869 STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 4870 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 4871 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 4872 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 4873 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 4874 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 4875 STAT_ATTR(ORDER_FALLBACK, order_fallback); 4876 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 4877 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 4878 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 4879 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 4880 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 4881 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 4882 #endif 4883 4884 static struct attribute *slab_attrs[] = { 4885 &slab_size_attr.attr, 4886 &object_size_attr.attr, 4887 &objs_per_slab_attr.attr, 4888 &order_attr.attr, 4889 &min_partial_attr.attr, 4890 &cpu_partial_attr.attr, 4891 &objects_attr.attr, 4892 &objects_partial_attr.attr, 4893 &partial_attr.attr, 4894 &cpu_slabs_attr.attr, 4895 &ctor_attr.attr, 4896 &aliases_attr.attr, 4897 &align_attr.attr, 4898 &hwcache_align_attr.attr, 4899 &reclaim_account_attr.attr, 4900 &destroy_by_rcu_attr.attr, 4901 &shrink_attr.attr, 4902 &reserved_attr.attr, 4903 &slabs_cpu_partial_attr.attr, 4904 #ifdef CONFIG_SLUB_DEBUG 4905 &total_objects_attr.attr, 4906 &slabs_attr.attr, 4907 &sanity_checks_attr.attr, 4908 &trace_attr.attr, 4909 &red_zone_attr.attr, 4910 &poison_attr.attr, 4911 &store_user_attr.attr, 4912 &validate_attr.attr, 4913 &alloc_calls_attr.attr, 4914 &free_calls_attr.attr, 4915 #endif 4916 #ifdef CONFIG_ZONE_DMA 4917 &cache_dma_attr.attr, 4918 #endif 4919 #ifdef CONFIG_NUMA 4920 &remote_node_defrag_ratio_attr.attr, 4921 #endif 4922 #ifdef CONFIG_SLUB_STATS 4923 &alloc_fastpath_attr.attr, 4924 &alloc_slowpath_attr.attr, 4925 &free_fastpath_attr.attr, 4926 &free_slowpath_attr.attr, 4927 &free_frozen_attr.attr, 4928 &free_add_partial_attr.attr, 4929 &free_remove_partial_attr.attr, 4930 &alloc_from_partial_attr.attr, 4931 &alloc_slab_attr.attr, 4932 &alloc_refill_attr.attr, 4933 &alloc_node_mismatch_attr.attr, 4934 &free_slab_attr.attr, 4935 &cpuslab_flush_attr.attr, 4936 &deactivate_full_attr.attr, 4937 &deactivate_empty_attr.attr, 4938 &deactivate_to_head_attr.attr, 4939 &deactivate_to_tail_attr.attr, 4940 &deactivate_remote_frees_attr.attr, 4941 &deactivate_bypass_attr.attr, 4942 &order_fallback_attr.attr, 4943 &cmpxchg_double_fail_attr.attr, 4944 &cmpxchg_double_cpu_fail_attr.attr, 4945 &cpu_partial_alloc_attr.attr, 4946 &cpu_partial_free_attr.attr, 4947 &cpu_partial_node_attr.attr, 4948 &cpu_partial_drain_attr.attr, 4949 #endif 4950 #ifdef CONFIG_FAILSLAB 4951 &failslab_attr.attr, 4952 #endif 4953 4954 NULL 4955 }; 4956 4957 static struct attribute_group slab_attr_group = { 4958 .attrs = slab_attrs, 4959 }; 4960 4961 static ssize_t slab_attr_show(struct kobject *kobj, 4962 struct attribute *attr, 4963 char *buf) 4964 { 4965 struct slab_attribute *attribute; 4966 struct kmem_cache *s; 4967 int err; 4968 4969 attribute = to_slab_attr(attr); 4970 s = to_slab(kobj); 4971 4972 if (!attribute->show) 4973 return -EIO; 4974 4975 err = attribute->show(s, buf); 4976 4977 return err; 4978 } 4979 4980 static ssize_t slab_attr_store(struct kobject *kobj, 4981 struct attribute *attr, 4982 const char *buf, size_t len) 4983 { 4984 struct slab_attribute *attribute; 4985 struct kmem_cache *s; 4986 int err; 4987 4988 attribute = to_slab_attr(attr); 4989 s = to_slab(kobj); 4990 4991 if (!attribute->store) 4992 return -EIO; 4993 4994 err = attribute->store(s, buf, len); 4995 #ifdef CONFIG_MEMCG_KMEM 4996 if (slab_state >= FULL && err >= 0 && is_root_cache(s)) { 4997 struct kmem_cache *c; 4998 4999 mutex_lock(&slab_mutex); 5000 if (s->max_attr_size < len) 5001 s->max_attr_size = len; 5002 5003 /* 5004 * This is a best effort propagation, so this function's return 5005 * value will be determined by the parent cache only. This is 5006 * basically because not all attributes will have a well 5007 * defined semantics for rollbacks - most of the actions will 5008 * have permanent effects. 5009 * 5010 * Returning the error value of any of the children that fail 5011 * is not 100 % defined, in the sense that users seeing the 5012 * error code won't be able to know anything about the state of 5013 * the cache. 5014 * 5015 * Only returning the error code for the parent cache at least 5016 * has well defined semantics. The cache being written to 5017 * directly either failed or succeeded, in which case we loop 5018 * through the descendants with best-effort propagation. 5019 */ 5020 for_each_memcg_cache(c, s) 5021 attribute->store(c, buf, len); 5022 mutex_unlock(&slab_mutex); 5023 } 5024 #endif 5025 return err; 5026 } 5027 5028 static void memcg_propagate_slab_attrs(struct kmem_cache *s) 5029 { 5030 #ifdef CONFIG_MEMCG_KMEM 5031 int i; 5032 char *buffer = NULL; 5033 struct kmem_cache *root_cache; 5034 5035 if (is_root_cache(s)) 5036 return; 5037 5038 root_cache = s->memcg_params.root_cache; 5039 5040 /* 5041 * This mean this cache had no attribute written. Therefore, no point 5042 * in copying default values around 5043 */ 5044 if (!root_cache->max_attr_size) 5045 return; 5046 5047 for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) { 5048 char mbuf[64]; 5049 char *buf; 5050 struct slab_attribute *attr = to_slab_attr(slab_attrs[i]); 5051 5052 if (!attr || !attr->store || !attr->show) 5053 continue; 5054 5055 /* 5056 * It is really bad that we have to allocate here, so we will 5057 * do it only as a fallback. If we actually allocate, though, 5058 * we can just use the allocated buffer until the end. 5059 * 5060 * Most of the slub attributes will tend to be very small in 5061 * size, but sysfs allows buffers up to a page, so they can 5062 * theoretically happen. 5063 */ 5064 if (buffer) 5065 buf = buffer; 5066 else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf)) 5067 buf = mbuf; 5068 else { 5069 buffer = (char *) get_zeroed_page(GFP_KERNEL); 5070 if (WARN_ON(!buffer)) 5071 continue; 5072 buf = buffer; 5073 } 5074 5075 attr->show(root_cache, buf); 5076 attr->store(s, buf, strlen(buf)); 5077 } 5078 5079 if (buffer) 5080 free_page((unsigned long)buffer); 5081 #endif 5082 } 5083 5084 static void kmem_cache_release(struct kobject *k) 5085 { 5086 slab_kmem_cache_release(to_slab(k)); 5087 } 5088 5089 static const struct sysfs_ops slab_sysfs_ops = { 5090 .show = slab_attr_show, 5091 .store = slab_attr_store, 5092 }; 5093 5094 static struct kobj_type slab_ktype = { 5095 .sysfs_ops = &slab_sysfs_ops, 5096 .release = kmem_cache_release, 5097 }; 5098 5099 static int uevent_filter(struct kset *kset, struct kobject *kobj) 5100 { 5101 struct kobj_type *ktype = get_ktype(kobj); 5102 5103 if (ktype == &slab_ktype) 5104 return 1; 5105 return 0; 5106 } 5107 5108 static const struct kset_uevent_ops slab_uevent_ops = { 5109 .filter = uevent_filter, 5110 }; 5111 5112 static struct kset *slab_kset; 5113 5114 static inline struct kset *cache_kset(struct kmem_cache *s) 5115 { 5116 #ifdef CONFIG_MEMCG_KMEM 5117 if (!is_root_cache(s)) 5118 return s->memcg_params.root_cache->memcg_kset; 5119 #endif 5120 return slab_kset; 5121 } 5122 5123 #define ID_STR_LENGTH 64 5124 5125 /* Create a unique string id for a slab cache: 5126 * 5127 * Format :[flags-]size 5128 */ 5129 static char *create_unique_id(struct kmem_cache *s) 5130 { 5131 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 5132 char *p = name; 5133 5134 BUG_ON(!name); 5135 5136 *p++ = ':'; 5137 /* 5138 * First flags affecting slabcache operations. We will only 5139 * get here for aliasable slabs so we do not need to support 5140 * too many flags. The flags here must cover all flags that 5141 * are matched during merging to guarantee that the id is 5142 * unique. 5143 */ 5144 if (s->flags & SLAB_CACHE_DMA) 5145 *p++ = 'd'; 5146 if (s->flags & SLAB_RECLAIM_ACCOUNT) 5147 *p++ = 'a'; 5148 if (s->flags & SLAB_DEBUG_FREE) 5149 *p++ = 'F'; 5150 if (!(s->flags & SLAB_NOTRACK)) 5151 *p++ = 't'; 5152 if (p != name + 1) 5153 *p++ = '-'; 5154 p += sprintf(p, "%07d", s->size); 5155 5156 BUG_ON(p > name + ID_STR_LENGTH - 1); 5157 return name; 5158 } 5159 5160 static int sysfs_slab_add(struct kmem_cache *s) 5161 { 5162 int err; 5163 const char *name; 5164 int unmergeable = slab_unmergeable(s); 5165 5166 if (unmergeable) { 5167 /* 5168 * Slabcache can never be merged so we can use the name proper. 5169 * This is typically the case for debug situations. In that 5170 * case we can catch duplicate names easily. 5171 */ 5172 sysfs_remove_link(&slab_kset->kobj, s->name); 5173 name = s->name; 5174 } else { 5175 /* 5176 * Create a unique name for the slab as a target 5177 * for the symlinks. 5178 */ 5179 name = create_unique_id(s); 5180 } 5181 5182 s->kobj.kset = cache_kset(s); 5183 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); 5184 if (err) 5185 goto out_put_kobj; 5186 5187 err = sysfs_create_group(&s->kobj, &slab_attr_group); 5188 if (err) 5189 goto out_del_kobj; 5190 5191 #ifdef CONFIG_MEMCG_KMEM 5192 if (is_root_cache(s)) { 5193 s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); 5194 if (!s->memcg_kset) { 5195 err = -ENOMEM; 5196 goto out_del_kobj; 5197 } 5198 } 5199 #endif 5200 5201 kobject_uevent(&s->kobj, KOBJ_ADD); 5202 if (!unmergeable) { 5203 /* Setup first alias */ 5204 sysfs_slab_alias(s, s->name); 5205 } 5206 out: 5207 if (!unmergeable) 5208 kfree(name); 5209 return err; 5210 out_del_kobj: 5211 kobject_del(&s->kobj); 5212 out_put_kobj: 5213 kobject_put(&s->kobj); 5214 goto out; 5215 } 5216 5217 void sysfs_slab_remove(struct kmem_cache *s) 5218 { 5219 if (slab_state < FULL) 5220 /* 5221 * Sysfs has not been setup yet so no need to remove the 5222 * cache from sysfs. 5223 */ 5224 return; 5225 5226 #ifdef CONFIG_MEMCG_KMEM 5227 kset_unregister(s->memcg_kset); 5228 #endif 5229 kobject_uevent(&s->kobj, KOBJ_REMOVE); 5230 kobject_del(&s->kobj); 5231 kobject_put(&s->kobj); 5232 } 5233 5234 /* 5235 * Need to buffer aliases during bootup until sysfs becomes 5236 * available lest we lose that information. 5237 */ 5238 struct saved_alias { 5239 struct kmem_cache *s; 5240 const char *name; 5241 struct saved_alias *next; 5242 }; 5243 5244 static struct saved_alias *alias_list; 5245 5246 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 5247 { 5248 struct saved_alias *al; 5249 5250 if (slab_state == FULL) { 5251 /* 5252 * If we have a leftover link then remove it. 5253 */ 5254 sysfs_remove_link(&slab_kset->kobj, name); 5255 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 5256 } 5257 5258 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 5259 if (!al) 5260 return -ENOMEM; 5261 5262 al->s = s; 5263 al->name = name; 5264 al->next = alias_list; 5265 alias_list = al; 5266 return 0; 5267 } 5268 5269 static int __init slab_sysfs_init(void) 5270 { 5271 struct kmem_cache *s; 5272 int err; 5273 5274 mutex_lock(&slab_mutex); 5275 5276 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); 5277 if (!slab_kset) { 5278 mutex_unlock(&slab_mutex); 5279 pr_err("Cannot register slab subsystem.\n"); 5280 return -ENOSYS; 5281 } 5282 5283 slab_state = FULL; 5284 5285 list_for_each_entry(s, &slab_caches, list) { 5286 err = sysfs_slab_add(s); 5287 if (err) 5288 pr_err("SLUB: Unable to add boot slab %s to sysfs\n", 5289 s->name); 5290 } 5291 5292 while (alias_list) { 5293 struct saved_alias *al = alias_list; 5294 5295 alias_list = alias_list->next; 5296 err = sysfs_slab_alias(al->s, al->name); 5297 if (err) 5298 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", 5299 al->name); 5300 kfree(al); 5301 } 5302 5303 mutex_unlock(&slab_mutex); 5304 resiliency_test(); 5305 return 0; 5306 } 5307 5308 __initcall(slab_sysfs_init); 5309 #endif /* CONFIG_SYSFS */ 5310 5311 /* 5312 * The /proc/slabinfo ABI 5313 */ 5314 #ifdef CONFIG_SLABINFO 5315 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 5316 { 5317 unsigned long nr_slabs = 0; 5318 unsigned long nr_objs = 0; 5319 unsigned long nr_free = 0; 5320 int node; 5321 struct kmem_cache_node *n; 5322 5323 for_each_kmem_cache_node(s, node, n) { 5324 nr_slabs += node_nr_slabs(n); 5325 nr_objs += node_nr_objs(n); 5326 nr_free += count_partial(n, count_free); 5327 } 5328 5329 sinfo->active_objs = nr_objs - nr_free; 5330 sinfo->num_objs = nr_objs; 5331 sinfo->active_slabs = nr_slabs; 5332 sinfo->num_slabs = nr_slabs; 5333 sinfo->objects_per_slab = oo_objects(s->oo); 5334 sinfo->cache_order = oo_order(s->oo); 5335 } 5336 5337 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s) 5338 { 5339 } 5340 5341 ssize_t slabinfo_write(struct file *file, const char __user *buffer, 5342 size_t count, loff_t *ppos) 5343 { 5344 return -EIO; 5345 } 5346 #endif /* CONFIG_SLABINFO */ 5347