1 /* 2 * SLUB: A slab allocator that limits cache line use instead of queuing 3 * objects in per cpu and per node lists. 4 * 5 * The allocator synchronizes using per slab locks or atomic operatios 6 * and only uses a centralized lock to manage a pool of partial slabs. 7 * 8 * (C) 2007 SGI, Christoph Lameter 9 * (C) 2011 Linux Foundation, Christoph Lameter 10 */ 11 12 #include <linux/mm.h> 13 #include <linux/swap.h> /* struct reclaim_state */ 14 #include <linux/module.h> 15 #include <linux/bit_spinlock.h> 16 #include <linux/interrupt.h> 17 #include <linux/bitops.h> 18 #include <linux/slab.h> 19 #include "slab.h" 20 #include <linux/proc_fs.h> 21 #include <linux/notifier.h> 22 #include <linux/seq_file.h> 23 #include <linux/kasan.h> 24 #include <linux/kmemcheck.h> 25 #include <linux/cpu.h> 26 #include <linux/cpuset.h> 27 #include <linux/mempolicy.h> 28 #include <linux/ctype.h> 29 #include <linux/debugobjects.h> 30 #include <linux/kallsyms.h> 31 #include <linux/memory.h> 32 #include <linux/math64.h> 33 #include <linux/fault-inject.h> 34 #include <linux/stacktrace.h> 35 #include <linux/prefetch.h> 36 #include <linux/memcontrol.h> 37 38 #include <trace/events/kmem.h> 39 40 #include "internal.h" 41 42 /* 43 * Lock order: 44 * 1. slab_mutex (Global Mutex) 45 * 2. node->list_lock 46 * 3. slab_lock(page) (Only on some arches and for debugging) 47 * 48 * slab_mutex 49 * 50 * The role of the slab_mutex is to protect the list of all the slabs 51 * and to synchronize major metadata changes to slab cache structures. 52 * 53 * The slab_lock is only used for debugging and on arches that do not 54 * have the ability to do a cmpxchg_double. It only protects the second 55 * double word in the page struct. Meaning 56 * A. page->freelist -> List of object free in a page 57 * B. page->counters -> Counters of objects 58 * C. page->frozen -> frozen state 59 * 60 * If a slab is frozen then it is exempt from list management. It is not 61 * on any list. The processor that froze the slab is the one who can 62 * perform list operations on the page. Other processors may put objects 63 * onto the freelist but the processor that froze the slab is the only 64 * one that can retrieve the objects from the page's freelist. 65 * 66 * The list_lock protects the partial and full list on each node and 67 * the partial slab counter. If taken then no new slabs may be added or 68 * removed from the lists nor make the number of partial slabs be modified. 69 * (Note that the total number of slabs is an atomic value that may be 70 * modified without taking the list lock). 71 * 72 * The list_lock is a centralized lock and thus we avoid taking it as 73 * much as possible. As long as SLUB does not have to handle partial 74 * slabs, operations can continue without any centralized lock. F.e. 75 * allocating a long series of objects that fill up slabs does not require 76 * the list lock. 77 * Interrupts are disabled during allocation and deallocation in order to 78 * make the slab allocator safe to use in the context of an irq. In addition 79 * interrupts are disabled to ensure that the processor does not change 80 * while handling per_cpu slabs, due to kernel preemption. 81 * 82 * SLUB assigns one slab for allocation to each processor. 83 * Allocations only occur from these slabs called cpu slabs. 84 * 85 * Slabs with free elements are kept on a partial list and during regular 86 * operations no list for full slabs is used. If an object in a full slab is 87 * freed then the slab will show up again on the partial lists. 88 * We track full slabs for debugging purposes though because otherwise we 89 * cannot scan all objects. 90 * 91 * Slabs are freed when they become empty. Teardown and setup is 92 * minimal so we rely on the page allocators per cpu caches for 93 * fast frees and allocs. 94 * 95 * Overloading of page flags that are otherwise used for LRU management. 96 * 97 * PageActive The slab is frozen and exempt from list processing. 98 * This means that the slab is dedicated to a purpose 99 * such as satisfying allocations for a specific 100 * processor. Objects may be freed in the slab while 101 * it is frozen but slab_free will then skip the usual 102 * list operations. It is up to the processor holding 103 * the slab to integrate the slab into the slab lists 104 * when the slab is no longer needed. 105 * 106 * One use of this flag is to mark slabs that are 107 * used for allocations. Then such a slab becomes a cpu 108 * slab. The cpu slab may be equipped with an additional 109 * freelist that allows lockless access to 110 * free objects in addition to the regular freelist 111 * that requires the slab lock. 112 * 113 * PageError Slab requires special handling due to debug 114 * options set. This moves slab handling out of 115 * the fast path and disables lockless freelists. 116 */ 117 118 static inline int kmem_cache_debug(struct kmem_cache *s) 119 { 120 #ifdef CONFIG_SLUB_DEBUG 121 return unlikely(s->flags & SLAB_DEBUG_FLAGS); 122 #else 123 return 0; 124 #endif 125 } 126 127 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) 128 { 129 #ifdef CONFIG_SLUB_CPU_PARTIAL 130 return !kmem_cache_debug(s); 131 #else 132 return false; 133 #endif 134 } 135 136 /* 137 * Issues still to be resolved: 138 * 139 * - Support PAGE_ALLOC_DEBUG. Should be easy to do. 140 * 141 * - Variable sizing of the per node arrays 142 */ 143 144 /* Enable to test recovery from slab corruption on boot */ 145 #undef SLUB_RESILIENCY_TEST 146 147 /* Enable to log cmpxchg failures */ 148 #undef SLUB_DEBUG_CMPXCHG 149 150 /* 151 * Mininum number of partial slabs. These will be left on the partial 152 * lists even if they are empty. kmem_cache_shrink may reclaim them. 153 */ 154 #define MIN_PARTIAL 5 155 156 /* 157 * Maximum number of desirable partial slabs. 158 * The existence of more partial slabs makes kmem_cache_shrink 159 * sort the partial list by the number of objects in use. 160 */ 161 #define MAX_PARTIAL 10 162 163 #define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ 164 SLAB_POISON | SLAB_STORE_USER) 165 166 /* 167 * Debugging flags that require metadata to be stored in the slab. These get 168 * disabled when slub_debug=O is used and a cache's min order increases with 169 * metadata. 170 */ 171 #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) 172 173 #define OO_SHIFT 16 174 #define OO_MASK ((1 << OO_SHIFT) - 1) 175 #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ 176 177 /* Internal SLUB flags */ 178 #define __OBJECT_POISON 0x80000000UL /* Poison object */ 179 #define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ 180 181 #ifdef CONFIG_SMP 182 static struct notifier_block slab_notifier; 183 #endif 184 185 /* 186 * Tracking user of a slab. 187 */ 188 #define TRACK_ADDRS_COUNT 16 189 struct track { 190 unsigned long addr; /* Called from address */ 191 #ifdef CONFIG_STACKTRACE 192 unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ 193 #endif 194 int cpu; /* Was running on cpu */ 195 int pid; /* Pid context */ 196 unsigned long when; /* When did the operation occur */ 197 }; 198 199 enum track_item { TRACK_ALLOC, TRACK_FREE }; 200 201 #ifdef CONFIG_SYSFS 202 static int sysfs_slab_add(struct kmem_cache *); 203 static int sysfs_slab_alias(struct kmem_cache *, const char *); 204 static void memcg_propagate_slab_attrs(struct kmem_cache *s); 205 #else 206 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } 207 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) 208 { return 0; } 209 static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } 210 #endif 211 212 static inline void stat(const struct kmem_cache *s, enum stat_item si) 213 { 214 #ifdef CONFIG_SLUB_STATS 215 /* 216 * The rmw is racy on a preemptible kernel but this is acceptable, so 217 * avoid this_cpu_add()'s irq-disable overhead. 218 */ 219 raw_cpu_inc(s->cpu_slab->stat[si]); 220 #endif 221 } 222 223 /******************************************************************** 224 * Core slab cache functions 225 *******************************************************************/ 226 227 /* Verify that a pointer has an address that is valid within a slab page */ 228 static inline int check_valid_pointer(struct kmem_cache *s, 229 struct page *page, const void *object) 230 { 231 void *base; 232 233 if (!object) 234 return 1; 235 236 base = page_address(page); 237 if (object < base || object >= base + page->objects * s->size || 238 (object - base) % s->size) { 239 return 0; 240 } 241 242 return 1; 243 } 244 245 static inline void *get_freepointer(struct kmem_cache *s, void *object) 246 { 247 return *(void **)(object + s->offset); 248 } 249 250 static void prefetch_freepointer(const struct kmem_cache *s, void *object) 251 { 252 prefetch(object + s->offset); 253 } 254 255 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) 256 { 257 void *p; 258 259 #ifdef CONFIG_DEBUG_PAGEALLOC 260 probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); 261 #else 262 p = get_freepointer(s, object); 263 #endif 264 return p; 265 } 266 267 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) 268 { 269 *(void **)(object + s->offset) = fp; 270 } 271 272 /* Loop over all objects in a slab */ 273 #define for_each_object(__p, __s, __addr, __objects) \ 274 for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ 275 __p += (__s)->size) 276 277 #define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ 278 for (__p = (__addr), __idx = 1; __idx <= __objects;\ 279 __p += (__s)->size, __idx++) 280 281 /* Determine object index from a given position */ 282 static inline int slab_index(void *p, struct kmem_cache *s, void *addr) 283 { 284 return (p - addr) / s->size; 285 } 286 287 static inline size_t slab_ksize(const struct kmem_cache *s) 288 { 289 #ifdef CONFIG_SLUB_DEBUG 290 /* 291 * Debugging requires use of the padding between object 292 * and whatever may come after it. 293 */ 294 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) 295 return s->object_size; 296 297 #endif 298 /* 299 * If we have the need to store the freelist pointer 300 * back there or track user information then we can 301 * only use the space before that information. 302 */ 303 if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) 304 return s->inuse; 305 /* 306 * Else we can use all the padding etc for the allocation 307 */ 308 return s->size; 309 } 310 311 static inline int order_objects(int order, unsigned long size, int reserved) 312 { 313 return ((PAGE_SIZE << order) - reserved) / size; 314 } 315 316 static inline struct kmem_cache_order_objects oo_make(int order, 317 unsigned long size, int reserved) 318 { 319 struct kmem_cache_order_objects x = { 320 (order << OO_SHIFT) + order_objects(order, size, reserved) 321 }; 322 323 return x; 324 } 325 326 static inline int oo_order(struct kmem_cache_order_objects x) 327 { 328 return x.x >> OO_SHIFT; 329 } 330 331 static inline int oo_objects(struct kmem_cache_order_objects x) 332 { 333 return x.x & OO_MASK; 334 } 335 336 /* 337 * Per slab locking using the pagelock 338 */ 339 static __always_inline void slab_lock(struct page *page) 340 { 341 bit_spin_lock(PG_locked, &page->flags); 342 } 343 344 static __always_inline void slab_unlock(struct page *page) 345 { 346 __bit_spin_unlock(PG_locked, &page->flags); 347 } 348 349 static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) 350 { 351 struct page tmp; 352 tmp.counters = counters_new; 353 /* 354 * page->counters can cover frozen/inuse/objects as well 355 * as page->_count. If we assign to ->counters directly 356 * we run the risk of losing updates to page->_count, so 357 * be careful and only assign to the fields we need. 358 */ 359 page->frozen = tmp.frozen; 360 page->inuse = tmp.inuse; 361 page->objects = tmp.objects; 362 } 363 364 /* Interrupts must be disabled (for the fallback code to work right) */ 365 static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, 366 void *freelist_old, unsigned long counters_old, 367 void *freelist_new, unsigned long counters_new, 368 const char *n) 369 { 370 VM_BUG_ON(!irqs_disabled()); 371 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 372 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 373 if (s->flags & __CMPXCHG_DOUBLE) { 374 if (cmpxchg_double(&page->freelist, &page->counters, 375 freelist_old, counters_old, 376 freelist_new, counters_new)) 377 return 1; 378 } else 379 #endif 380 { 381 slab_lock(page); 382 if (page->freelist == freelist_old && 383 page->counters == counters_old) { 384 page->freelist = freelist_new; 385 set_page_slub_counters(page, counters_new); 386 slab_unlock(page); 387 return 1; 388 } 389 slab_unlock(page); 390 } 391 392 cpu_relax(); 393 stat(s, CMPXCHG_DOUBLE_FAIL); 394 395 #ifdef SLUB_DEBUG_CMPXCHG 396 pr_info("%s %s: cmpxchg double redo ", n, s->name); 397 #endif 398 399 return 0; 400 } 401 402 static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, 403 void *freelist_old, unsigned long counters_old, 404 void *freelist_new, unsigned long counters_new, 405 const char *n) 406 { 407 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 408 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 409 if (s->flags & __CMPXCHG_DOUBLE) { 410 if (cmpxchg_double(&page->freelist, &page->counters, 411 freelist_old, counters_old, 412 freelist_new, counters_new)) 413 return 1; 414 } else 415 #endif 416 { 417 unsigned long flags; 418 419 local_irq_save(flags); 420 slab_lock(page); 421 if (page->freelist == freelist_old && 422 page->counters == counters_old) { 423 page->freelist = freelist_new; 424 set_page_slub_counters(page, counters_new); 425 slab_unlock(page); 426 local_irq_restore(flags); 427 return 1; 428 } 429 slab_unlock(page); 430 local_irq_restore(flags); 431 } 432 433 cpu_relax(); 434 stat(s, CMPXCHG_DOUBLE_FAIL); 435 436 #ifdef SLUB_DEBUG_CMPXCHG 437 pr_info("%s %s: cmpxchg double redo ", n, s->name); 438 #endif 439 440 return 0; 441 } 442 443 #ifdef CONFIG_SLUB_DEBUG 444 /* 445 * Determine a map of object in use on a page. 446 * 447 * Node listlock must be held to guarantee that the page does 448 * not vanish from under us. 449 */ 450 static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) 451 { 452 void *p; 453 void *addr = page_address(page); 454 455 for (p = page->freelist; p; p = get_freepointer(s, p)) 456 set_bit(slab_index(p, s, addr), map); 457 } 458 459 /* 460 * Debug settings: 461 */ 462 #ifdef CONFIG_SLUB_DEBUG_ON 463 static int slub_debug = DEBUG_DEFAULT_FLAGS; 464 #else 465 static int slub_debug; 466 #endif 467 468 static char *slub_debug_slabs; 469 static int disable_higher_order_debug; 470 471 /* 472 * slub is about to manipulate internal object metadata. This memory lies 473 * outside the range of the allocated object, so accessing it would normally 474 * be reported by kasan as a bounds error. metadata_access_enable() is used 475 * to tell kasan that these accesses are OK. 476 */ 477 static inline void metadata_access_enable(void) 478 { 479 kasan_disable_current(); 480 } 481 482 static inline void metadata_access_disable(void) 483 { 484 kasan_enable_current(); 485 } 486 487 /* 488 * Object debugging 489 */ 490 static void print_section(char *text, u8 *addr, unsigned int length) 491 { 492 metadata_access_enable(); 493 print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, 494 length, 1); 495 metadata_access_disable(); 496 } 497 498 static struct track *get_track(struct kmem_cache *s, void *object, 499 enum track_item alloc) 500 { 501 struct track *p; 502 503 if (s->offset) 504 p = object + s->offset + sizeof(void *); 505 else 506 p = object + s->inuse; 507 508 return p + alloc; 509 } 510 511 static void set_track(struct kmem_cache *s, void *object, 512 enum track_item alloc, unsigned long addr) 513 { 514 struct track *p = get_track(s, object, alloc); 515 516 if (addr) { 517 #ifdef CONFIG_STACKTRACE 518 struct stack_trace trace; 519 int i; 520 521 trace.nr_entries = 0; 522 trace.max_entries = TRACK_ADDRS_COUNT; 523 trace.entries = p->addrs; 524 trace.skip = 3; 525 metadata_access_enable(); 526 save_stack_trace(&trace); 527 metadata_access_disable(); 528 529 /* See rant in lockdep.c */ 530 if (trace.nr_entries != 0 && 531 trace.entries[trace.nr_entries - 1] == ULONG_MAX) 532 trace.nr_entries--; 533 534 for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) 535 p->addrs[i] = 0; 536 #endif 537 p->addr = addr; 538 p->cpu = smp_processor_id(); 539 p->pid = current->pid; 540 p->when = jiffies; 541 } else 542 memset(p, 0, sizeof(struct track)); 543 } 544 545 static void init_tracking(struct kmem_cache *s, void *object) 546 { 547 if (!(s->flags & SLAB_STORE_USER)) 548 return; 549 550 set_track(s, object, TRACK_FREE, 0UL); 551 set_track(s, object, TRACK_ALLOC, 0UL); 552 } 553 554 static void print_track(const char *s, struct track *t) 555 { 556 if (!t->addr) 557 return; 558 559 pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n", 560 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); 561 #ifdef CONFIG_STACKTRACE 562 { 563 int i; 564 for (i = 0; i < TRACK_ADDRS_COUNT; i++) 565 if (t->addrs[i]) 566 pr_err("\t%pS\n", (void *)t->addrs[i]); 567 else 568 break; 569 } 570 #endif 571 } 572 573 static void print_tracking(struct kmem_cache *s, void *object) 574 { 575 if (!(s->flags & SLAB_STORE_USER)) 576 return; 577 578 print_track("Allocated", get_track(s, object, TRACK_ALLOC)); 579 print_track("Freed", get_track(s, object, TRACK_FREE)); 580 } 581 582 static void print_page_info(struct page *page) 583 { 584 pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", 585 page, page->objects, page->inuse, page->freelist, page->flags); 586 587 } 588 589 static void slab_bug(struct kmem_cache *s, char *fmt, ...) 590 { 591 struct va_format vaf; 592 va_list args; 593 594 va_start(args, fmt); 595 vaf.fmt = fmt; 596 vaf.va = &args; 597 pr_err("=============================================================================\n"); 598 pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); 599 pr_err("-----------------------------------------------------------------------------\n\n"); 600 601 add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); 602 va_end(args); 603 } 604 605 static void slab_fix(struct kmem_cache *s, char *fmt, ...) 606 { 607 struct va_format vaf; 608 va_list args; 609 610 va_start(args, fmt); 611 vaf.fmt = fmt; 612 vaf.va = &args; 613 pr_err("FIX %s: %pV\n", s->name, &vaf); 614 va_end(args); 615 } 616 617 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) 618 { 619 unsigned int off; /* Offset of last byte */ 620 u8 *addr = page_address(page); 621 622 print_tracking(s, p); 623 624 print_page_info(page); 625 626 pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", 627 p, p - addr, get_freepointer(s, p)); 628 629 if (p > addr + 16) 630 print_section("Bytes b4 ", p - 16, 16); 631 632 print_section("Object ", p, min_t(unsigned long, s->object_size, 633 PAGE_SIZE)); 634 if (s->flags & SLAB_RED_ZONE) 635 print_section("Redzone ", p + s->object_size, 636 s->inuse - s->object_size); 637 638 if (s->offset) 639 off = s->offset + sizeof(void *); 640 else 641 off = s->inuse; 642 643 if (s->flags & SLAB_STORE_USER) 644 off += 2 * sizeof(struct track); 645 646 if (off != s->size) 647 /* Beginning of the filler is the free pointer */ 648 print_section("Padding ", p + off, s->size - off); 649 650 dump_stack(); 651 } 652 653 void object_err(struct kmem_cache *s, struct page *page, 654 u8 *object, char *reason) 655 { 656 slab_bug(s, "%s", reason); 657 print_trailer(s, page, object); 658 } 659 660 static void slab_err(struct kmem_cache *s, struct page *page, 661 const char *fmt, ...) 662 { 663 va_list args; 664 char buf[100]; 665 666 va_start(args, fmt); 667 vsnprintf(buf, sizeof(buf), fmt, args); 668 va_end(args); 669 slab_bug(s, "%s", buf); 670 print_page_info(page); 671 dump_stack(); 672 } 673 674 static void init_object(struct kmem_cache *s, void *object, u8 val) 675 { 676 u8 *p = object; 677 678 if (s->flags & __OBJECT_POISON) { 679 memset(p, POISON_FREE, s->object_size - 1); 680 p[s->object_size - 1] = POISON_END; 681 } 682 683 if (s->flags & SLAB_RED_ZONE) 684 memset(p + s->object_size, val, s->inuse - s->object_size); 685 } 686 687 static void restore_bytes(struct kmem_cache *s, char *message, u8 data, 688 void *from, void *to) 689 { 690 slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); 691 memset(from, data, to - from); 692 } 693 694 static int check_bytes_and_report(struct kmem_cache *s, struct page *page, 695 u8 *object, char *what, 696 u8 *start, unsigned int value, unsigned int bytes) 697 { 698 u8 *fault; 699 u8 *end; 700 701 metadata_access_enable(); 702 fault = memchr_inv(start, value, bytes); 703 metadata_access_disable(); 704 if (!fault) 705 return 1; 706 707 end = start + bytes; 708 while (end > fault && end[-1] == value) 709 end--; 710 711 slab_bug(s, "%s overwritten", what); 712 pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", 713 fault, end - 1, fault[0], value); 714 print_trailer(s, page, object); 715 716 restore_bytes(s, what, value, fault, end); 717 return 0; 718 } 719 720 /* 721 * Object layout: 722 * 723 * object address 724 * Bytes of the object to be managed. 725 * If the freepointer may overlay the object then the free 726 * pointer is the first word of the object. 727 * 728 * Poisoning uses 0x6b (POISON_FREE) and the last byte is 729 * 0xa5 (POISON_END) 730 * 731 * object + s->object_size 732 * Padding to reach word boundary. This is also used for Redzoning. 733 * Padding is extended by another word if Redzoning is enabled and 734 * object_size == inuse. 735 * 736 * We fill with 0xbb (RED_INACTIVE) for inactive objects and with 737 * 0xcc (RED_ACTIVE) for objects in use. 738 * 739 * object + s->inuse 740 * Meta data starts here. 741 * 742 * A. Free pointer (if we cannot overwrite object on free) 743 * B. Tracking data for SLAB_STORE_USER 744 * C. Padding to reach required alignment boundary or at mininum 745 * one word if debugging is on to be able to detect writes 746 * before the word boundary. 747 * 748 * Padding is done using 0x5a (POISON_INUSE) 749 * 750 * object + s->size 751 * Nothing is used beyond s->size. 752 * 753 * If slabcaches are merged then the object_size and inuse boundaries are mostly 754 * ignored. And therefore no slab options that rely on these boundaries 755 * may be used with merged slabcaches. 756 */ 757 758 static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) 759 { 760 unsigned long off = s->inuse; /* The end of info */ 761 762 if (s->offset) 763 /* Freepointer is placed after the object. */ 764 off += sizeof(void *); 765 766 if (s->flags & SLAB_STORE_USER) 767 /* We also have user information there */ 768 off += 2 * sizeof(struct track); 769 770 if (s->size == off) 771 return 1; 772 773 return check_bytes_and_report(s, page, p, "Object padding", 774 p + off, POISON_INUSE, s->size - off); 775 } 776 777 /* Check the pad bytes at the end of a slab page */ 778 static int slab_pad_check(struct kmem_cache *s, struct page *page) 779 { 780 u8 *start; 781 u8 *fault; 782 u8 *end; 783 int length; 784 int remainder; 785 786 if (!(s->flags & SLAB_POISON)) 787 return 1; 788 789 start = page_address(page); 790 length = (PAGE_SIZE << compound_order(page)) - s->reserved; 791 end = start + length; 792 remainder = length % s->size; 793 if (!remainder) 794 return 1; 795 796 metadata_access_enable(); 797 fault = memchr_inv(end - remainder, POISON_INUSE, remainder); 798 metadata_access_disable(); 799 if (!fault) 800 return 1; 801 while (end > fault && end[-1] == POISON_INUSE) 802 end--; 803 804 slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); 805 print_section("Padding ", end - remainder, remainder); 806 807 restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); 808 return 0; 809 } 810 811 static int check_object(struct kmem_cache *s, struct page *page, 812 void *object, u8 val) 813 { 814 u8 *p = object; 815 u8 *endobject = object + s->object_size; 816 817 if (s->flags & SLAB_RED_ZONE) { 818 if (!check_bytes_and_report(s, page, object, "Redzone", 819 endobject, val, s->inuse - s->object_size)) 820 return 0; 821 } else { 822 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { 823 check_bytes_and_report(s, page, p, "Alignment padding", 824 endobject, POISON_INUSE, 825 s->inuse - s->object_size); 826 } 827 } 828 829 if (s->flags & SLAB_POISON) { 830 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && 831 (!check_bytes_and_report(s, page, p, "Poison", p, 832 POISON_FREE, s->object_size - 1) || 833 !check_bytes_and_report(s, page, p, "Poison", 834 p + s->object_size - 1, POISON_END, 1))) 835 return 0; 836 /* 837 * check_pad_bytes cleans up on its own. 838 */ 839 check_pad_bytes(s, page, p); 840 } 841 842 if (!s->offset && val == SLUB_RED_ACTIVE) 843 /* 844 * Object and freepointer overlap. Cannot check 845 * freepointer while object is allocated. 846 */ 847 return 1; 848 849 /* Check free pointer validity */ 850 if (!check_valid_pointer(s, page, get_freepointer(s, p))) { 851 object_err(s, page, p, "Freepointer corrupt"); 852 /* 853 * No choice but to zap it and thus lose the remainder 854 * of the free objects in this slab. May cause 855 * another error because the object count is now wrong. 856 */ 857 set_freepointer(s, p, NULL); 858 return 0; 859 } 860 return 1; 861 } 862 863 static int check_slab(struct kmem_cache *s, struct page *page) 864 { 865 int maxobj; 866 867 VM_BUG_ON(!irqs_disabled()); 868 869 if (!PageSlab(page)) { 870 slab_err(s, page, "Not a valid slab page"); 871 return 0; 872 } 873 874 maxobj = order_objects(compound_order(page), s->size, s->reserved); 875 if (page->objects > maxobj) { 876 slab_err(s, page, "objects %u > max %u", 877 page->objects, maxobj); 878 return 0; 879 } 880 if (page->inuse > page->objects) { 881 slab_err(s, page, "inuse %u > max %u", 882 page->inuse, page->objects); 883 return 0; 884 } 885 /* Slab_pad_check fixes things up after itself */ 886 slab_pad_check(s, page); 887 return 1; 888 } 889 890 /* 891 * Determine if a certain object on a page is on the freelist. Must hold the 892 * slab lock to guarantee that the chains are in a consistent state. 893 */ 894 static int on_freelist(struct kmem_cache *s, struct page *page, void *search) 895 { 896 int nr = 0; 897 void *fp; 898 void *object = NULL; 899 int max_objects; 900 901 fp = page->freelist; 902 while (fp && nr <= page->objects) { 903 if (fp == search) 904 return 1; 905 if (!check_valid_pointer(s, page, fp)) { 906 if (object) { 907 object_err(s, page, object, 908 "Freechain corrupt"); 909 set_freepointer(s, object, NULL); 910 } else { 911 slab_err(s, page, "Freepointer corrupt"); 912 page->freelist = NULL; 913 page->inuse = page->objects; 914 slab_fix(s, "Freelist cleared"); 915 return 0; 916 } 917 break; 918 } 919 object = fp; 920 fp = get_freepointer(s, object); 921 nr++; 922 } 923 924 max_objects = order_objects(compound_order(page), s->size, s->reserved); 925 if (max_objects > MAX_OBJS_PER_PAGE) 926 max_objects = MAX_OBJS_PER_PAGE; 927 928 if (page->objects != max_objects) { 929 slab_err(s, page, "Wrong number of objects. Found %d but " 930 "should be %d", page->objects, max_objects); 931 page->objects = max_objects; 932 slab_fix(s, "Number of objects adjusted."); 933 } 934 if (page->inuse != page->objects - nr) { 935 slab_err(s, page, "Wrong object count. Counter is %d but " 936 "counted were %d", page->inuse, page->objects - nr); 937 page->inuse = page->objects - nr; 938 slab_fix(s, "Object count adjusted."); 939 } 940 return search == NULL; 941 } 942 943 static void trace(struct kmem_cache *s, struct page *page, void *object, 944 int alloc) 945 { 946 if (s->flags & SLAB_TRACE) { 947 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", 948 s->name, 949 alloc ? "alloc" : "free", 950 object, page->inuse, 951 page->freelist); 952 953 if (!alloc) 954 print_section("Object ", (void *)object, 955 s->object_size); 956 957 dump_stack(); 958 } 959 } 960 961 /* 962 * Tracking of fully allocated slabs for debugging purposes. 963 */ 964 static void add_full(struct kmem_cache *s, 965 struct kmem_cache_node *n, struct page *page) 966 { 967 if (!(s->flags & SLAB_STORE_USER)) 968 return; 969 970 lockdep_assert_held(&n->list_lock); 971 list_add(&page->lru, &n->full); 972 } 973 974 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) 975 { 976 if (!(s->flags & SLAB_STORE_USER)) 977 return; 978 979 lockdep_assert_held(&n->list_lock); 980 list_del(&page->lru); 981 } 982 983 /* Tracking of the number of slabs for debugging purposes */ 984 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 985 { 986 struct kmem_cache_node *n = get_node(s, node); 987 988 return atomic_long_read(&n->nr_slabs); 989 } 990 991 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 992 { 993 return atomic_long_read(&n->nr_slabs); 994 } 995 996 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) 997 { 998 struct kmem_cache_node *n = get_node(s, node); 999 1000 /* 1001 * May be called early in order to allocate a slab for the 1002 * kmem_cache_node structure. Solve the chicken-egg 1003 * dilemma by deferring the increment of the count during 1004 * bootstrap (see early_kmem_cache_node_alloc). 1005 */ 1006 if (likely(n)) { 1007 atomic_long_inc(&n->nr_slabs); 1008 atomic_long_add(objects, &n->total_objects); 1009 } 1010 } 1011 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) 1012 { 1013 struct kmem_cache_node *n = get_node(s, node); 1014 1015 atomic_long_dec(&n->nr_slabs); 1016 atomic_long_sub(objects, &n->total_objects); 1017 } 1018 1019 /* Object debug checks for alloc/free paths */ 1020 static void setup_object_debug(struct kmem_cache *s, struct page *page, 1021 void *object) 1022 { 1023 if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) 1024 return; 1025 1026 init_object(s, object, SLUB_RED_INACTIVE); 1027 init_tracking(s, object); 1028 } 1029 1030 static noinline int alloc_debug_processing(struct kmem_cache *s, 1031 struct page *page, 1032 void *object, unsigned long addr) 1033 { 1034 if (!check_slab(s, page)) 1035 goto bad; 1036 1037 if (!check_valid_pointer(s, page, object)) { 1038 object_err(s, page, object, "Freelist Pointer check fails"); 1039 goto bad; 1040 } 1041 1042 if (!check_object(s, page, object, SLUB_RED_INACTIVE)) 1043 goto bad; 1044 1045 /* Success perform special debug activities for allocs */ 1046 if (s->flags & SLAB_STORE_USER) 1047 set_track(s, object, TRACK_ALLOC, addr); 1048 trace(s, page, object, 1); 1049 init_object(s, object, SLUB_RED_ACTIVE); 1050 return 1; 1051 1052 bad: 1053 if (PageSlab(page)) { 1054 /* 1055 * If this is a slab page then lets do the best we can 1056 * to avoid issues in the future. Marking all objects 1057 * as used avoids touching the remaining objects. 1058 */ 1059 slab_fix(s, "Marking all objects used"); 1060 page->inuse = page->objects; 1061 page->freelist = NULL; 1062 } 1063 return 0; 1064 } 1065 1066 static noinline struct kmem_cache_node *free_debug_processing( 1067 struct kmem_cache *s, struct page *page, void *object, 1068 unsigned long addr, unsigned long *flags) 1069 { 1070 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1071 1072 spin_lock_irqsave(&n->list_lock, *flags); 1073 slab_lock(page); 1074 1075 if (!check_slab(s, page)) 1076 goto fail; 1077 1078 if (!check_valid_pointer(s, page, object)) { 1079 slab_err(s, page, "Invalid object pointer 0x%p", object); 1080 goto fail; 1081 } 1082 1083 if (on_freelist(s, page, object)) { 1084 object_err(s, page, object, "Object already free"); 1085 goto fail; 1086 } 1087 1088 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) 1089 goto out; 1090 1091 if (unlikely(s != page->slab_cache)) { 1092 if (!PageSlab(page)) { 1093 slab_err(s, page, "Attempt to free object(0x%p) " 1094 "outside of slab", object); 1095 } else if (!page->slab_cache) { 1096 pr_err("SLUB <none>: no slab for object 0x%p.\n", 1097 object); 1098 dump_stack(); 1099 } else 1100 object_err(s, page, object, 1101 "page slab pointer corrupt."); 1102 goto fail; 1103 } 1104 1105 if (s->flags & SLAB_STORE_USER) 1106 set_track(s, object, TRACK_FREE, addr); 1107 trace(s, page, object, 0); 1108 init_object(s, object, SLUB_RED_INACTIVE); 1109 out: 1110 slab_unlock(page); 1111 /* 1112 * Keep node_lock to preserve integrity 1113 * until the object is actually freed 1114 */ 1115 return n; 1116 1117 fail: 1118 slab_unlock(page); 1119 spin_unlock_irqrestore(&n->list_lock, *flags); 1120 slab_fix(s, "Object at 0x%p not freed", object); 1121 return NULL; 1122 } 1123 1124 static int __init setup_slub_debug(char *str) 1125 { 1126 slub_debug = DEBUG_DEFAULT_FLAGS; 1127 if (*str++ != '=' || !*str) 1128 /* 1129 * No options specified. Switch on full debugging. 1130 */ 1131 goto out; 1132 1133 if (*str == ',') 1134 /* 1135 * No options but restriction on slabs. This means full 1136 * debugging for slabs matching a pattern. 1137 */ 1138 goto check_slabs; 1139 1140 if (tolower(*str) == 'o') { 1141 /* 1142 * Avoid enabling debugging on caches if its minimum order 1143 * would increase as a result. 1144 */ 1145 disable_higher_order_debug = 1; 1146 goto out; 1147 } 1148 1149 slub_debug = 0; 1150 if (*str == '-') 1151 /* 1152 * Switch off all debugging measures. 1153 */ 1154 goto out; 1155 1156 /* 1157 * Determine which debug features should be switched on 1158 */ 1159 for (; *str && *str != ','; str++) { 1160 switch (tolower(*str)) { 1161 case 'f': 1162 slub_debug |= SLAB_DEBUG_FREE; 1163 break; 1164 case 'z': 1165 slub_debug |= SLAB_RED_ZONE; 1166 break; 1167 case 'p': 1168 slub_debug |= SLAB_POISON; 1169 break; 1170 case 'u': 1171 slub_debug |= SLAB_STORE_USER; 1172 break; 1173 case 't': 1174 slub_debug |= SLAB_TRACE; 1175 break; 1176 case 'a': 1177 slub_debug |= SLAB_FAILSLAB; 1178 break; 1179 default: 1180 pr_err("slub_debug option '%c' unknown. skipped\n", 1181 *str); 1182 } 1183 } 1184 1185 check_slabs: 1186 if (*str == ',') 1187 slub_debug_slabs = str + 1; 1188 out: 1189 return 1; 1190 } 1191 1192 __setup("slub_debug", setup_slub_debug); 1193 1194 unsigned long kmem_cache_flags(unsigned long object_size, 1195 unsigned long flags, const char *name, 1196 void (*ctor)(void *)) 1197 { 1198 /* 1199 * Enable debugging if selected on the kernel commandline. 1200 */ 1201 if (slub_debug && (!slub_debug_slabs || (name && 1202 !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) 1203 flags |= slub_debug; 1204 1205 return flags; 1206 } 1207 #else 1208 static inline void setup_object_debug(struct kmem_cache *s, 1209 struct page *page, void *object) {} 1210 1211 static inline int alloc_debug_processing(struct kmem_cache *s, 1212 struct page *page, void *object, unsigned long addr) { return 0; } 1213 1214 static inline struct kmem_cache_node *free_debug_processing( 1215 struct kmem_cache *s, struct page *page, void *object, 1216 unsigned long addr, unsigned long *flags) { return NULL; } 1217 1218 static inline int slab_pad_check(struct kmem_cache *s, struct page *page) 1219 { return 1; } 1220 static inline int check_object(struct kmem_cache *s, struct page *page, 1221 void *object, u8 val) { return 1; } 1222 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, 1223 struct page *page) {} 1224 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, 1225 struct page *page) {} 1226 unsigned long kmem_cache_flags(unsigned long object_size, 1227 unsigned long flags, const char *name, 1228 void (*ctor)(void *)) 1229 { 1230 return flags; 1231 } 1232 #define slub_debug 0 1233 1234 #define disable_higher_order_debug 0 1235 1236 static inline unsigned long slabs_node(struct kmem_cache *s, int node) 1237 { return 0; } 1238 static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) 1239 { return 0; } 1240 static inline void inc_slabs_node(struct kmem_cache *s, int node, 1241 int objects) {} 1242 static inline void dec_slabs_node(struct kmem_cache *s, int node, 1243 int objects) {} 1244 1245 #endif /* CONFIG_SLUB_DEBUG */ 1246 1247 /* 1248 * Hooks for other subsystems that check memory allocations. In a typical 1249 * production configuration these hooks all should produce no code at all. 1250 */ 1251 static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) 1252 { 1253 kmemleak_alloc(ptr, size, 1, flags); 1254 kasan_kmalloc_large(ptr, size); 1255 } 1256 1257 static inline void kfree_hook(const void *x) 1258 { 1259 kmemleak_free(x); 1260 kasan_kfree_large(x); 1261 } 1262 1263 static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, 1264 gfp_t flags) 1265 { 1266 flags &= gfp_allowed_mask; 1267 lockdep_trace_alloc(flags); 1268 might_sleep_if(flags & __GFP_WAIT); 1269 1270 if (should_failslab(s->object_size, flags, s->flags)) 1271 return NULL; 1272 1273 return memcg_kmem_get_cache(s, flags); 1274 } 1275 1276 static inline void slab_post_alloc_hook(struct kmem_cache *s, 1277 gfp_t flags, void *object) 1278 { 1279 flags &= gfp_allowed_mask; 1280 kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); 1281 kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); 1282 memcg_kmem_put_cache(s); 1283 kasan_slab_alloc(s, object); 1284 } 1285 1286 static inline void slab_free_hook(struct kmem_cache *s, void *x) 1287 { 1288 kmemleak_free_recursive(x, s->flags); 1289 1290 /* 1291 * Trouble is that we may no longer disable interrupts in the fast path 1292 * So in order to make the debug calls that expect irqs to be 1293 * disabled we need to disable interrupts temporarily. 1294 */ 1295 #if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) 1296 { 1297 unsigned long flags; 1298 1299 local_irq_save(flags); 1300 kmemcheck_slab_free(s, x, s->object_size); 1301 debug_check_no_locks_freed(x, s->object_size); 1302 local_irq_restore(flags); 1303 } 1304 #endif 1305 if (!(s->flags & SLAB_DEBUG_OBJECTS)) 1306 debug_check_no_obj_freed(x, s->object_size); 1307 1308 kasan_slab_free(s, x); 1309 } 1310 1311 /* 1312 * Slab allocation and freeing 1313 */ 1314 static inline struct page *alloc_slab_page(struct kmem_cache *s, 1315 gfp_t flags, int node, struct kmem_cache_order_objects oo) 1316 { 1317 struct page *page; 1318 int order = oo_order(oo); 1319 1320 flags |= __GFP_NOTRACK; 1321 1322 if (memcg_charge_slab(s, flags, order)) 1323 return NULL; 1324 1325 if (node == NUMA_NO_NODE) 1326 page = alloc_pages(flags, order); 1327 else 1328 page = alloc_pages_exact_node(node, flags, order); 1329 1330 if (!page) 1331 memcg_uncharge_slab(s, order); 1332 1333 return page; 1334 } 1335 1336 static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) 1337 { 1338 struct page *page; 1339 struct kmem_cache_order_objects oo = s->oo; 1340 gfp_t alloc_gfp; 1341 1342 flags &= gfp_allowed_mask; 1343 1344 if (flags & __GFP_WAIT) 1345 local_irq_enable(); 1346 1347 flags |= s->allocflags; 1348 1349 /* 1350 * Let the initial higher-order allocation fail under memory pressure 1351 * so we fall-back to the minimum order allocation. 1352 */ 1353 alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; 1354 1355 page = alloc_slab_page(s, alloc_gfp, node, oo); 1356 if (unlikely(!page)) { 1357 oo = s->min; 1358 alloc_gfp = flags; 1359 /* 1360 * Allocation may have failed due to fragmentation. 1361 * Try a lower order alloc if possible 1362 */ 1363 page = alloc_slab_page(s, alloc_gfp, node, oo); 1364 1365 if (page) 1366 stat(s, ORDER_FALLBACK); 1367 } 1368 1369 if (kmemcheck_enabled && page 1370 && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { 1371 int pages = 1 << oo_order(oo); 1372 1373 kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); 1374 1375 /* 1376 * Objects from caches that have a constructor don't get 1377 * cleared when they're allocated, so we need to do it here. 1378 */ 1379 if (s->ctor) 1380 kmemcheck_mark_uninitialized_pages(page, pages); 1381 else 1382 kmemcheck_mark_unallocated_pages(page, pages); 1383 } 1384 1385 if (flags & __GFP_WAIT) 1386 local_irq_disable(); 1387 if (!page) 1388 return NULL; 1389 1390 page->objects = oo_objects(oo); 1391 mod_zone_page_state(page_zone(page), 1392 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1393 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1394 1 << oo_order(oo)); 1395 1396 return page; 1397 } 1398 1399 static void setup_object(struct kmem_cache *s, struct page *page, 1400 void *object) 1401 { 1402 setup_object_debug(s, page, object); 1403 if (unlikely(s->ctor)) { 1404 kasan_unpoison_object_data(s, object); 1405 s->ctor(object); 1406 kasan_poison_object_data(s, object); 1407 } 1408 } 1409 1410 static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) 1411 { 1412 struct page *page; 1413 void *start; 1414 void *p; 1415 int order; 1416 int idx; 1417 1418 if (unlikely(flags & GFP_SLAB_BUG_MASK)) { 1419 pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); 1420 BUG(); 1421 } 1422 1423 page = allocate_slab(s, 1424 flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); 1425 if (!page) 1426 goto out; 1427 1428 order = compound_order(page); 1429 inc_slabs_node(s, page_to_nid(page), page->objects); 1430 page->slab_cache = s; 1431 __SetPageSlab(page); 1432 if (page->pfmemalloc) 1433 SetPageSlabPfmemalloc(page); 1434 1435 start = page_address(page); 1436 1437 if (unlikely(s->flags & SLAB_POISON)) 1438 memset(start, POISON_INUSE, PAGE_SIZE << order); 1439 1440 kasan_poison_slab(page); 1441 1442 for_each_object_idx(p, idx, s, start, page->objects) { 1443 setup_object(s, page, p); 1444 if (likely(idx < page->objects)) 1445 set_freepointer(s, p, p + s->size); 1446 else 1447 set_freepointer(s, p, NULL); 1448 } 1449 1450 page->freelist = start; 1451 page->inuse = page->objects; 1452 page->frozen = 1; 1453 out: 1454 return page; 1455 } 1456 1457 static void __free_slab(struct kmem_cache *s, struct page *page) 1458 { 1459 int order = compound_order(page); 1460 int pages = 1 << order; 1461 1462 if (kmem_cache_debug(s)) { 1463 void *p; 1464 1465 slab_pad_check(s, page); 1466 for_each_object(p, s, page_address(page), 1467 page->objects) 1468 check_object(s, page, p, SLUB_RED_INACTIVE); 1469 } 1470 1471 kmemcheck_free_shadow(page, compound_order(page)); 1472 1473 mod_zone_page_state(page_zone(page), 1474 (s->flags & SLAB_RECLAIM_ACCOUNT) ? 1475 NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, 1476 -pages); 1477 1478 __ClearPageSlabPfmemalloc(page); 1479 __ClearPageSlab(page); 1480 1481 page_mapcount_reset(page); 1482 if (current->reclaim_state) 1483 current->reclaim_state->reclaimed_slab += pages; 1484 __free_pages(page, order); 1485 memcg_uncharge_slab(s, order); 1486 } 1487 1488 #define need_reserve_slab_rcu \ 1489 (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) 1490 1491 static void rcu_free_slab(struct rcu_head *h) 1492 { 1493 struct page *page; 1494 1495 if (need_reserve_slab_rcu) 1496 page = virt_to_head_page(h); 1497 else 1498 page = container_of((struct list_head *)h, struct page, lru); 1499 1500 __free_slab(page->slab_cache, page); 1501 } 1502 1503 static void free_slab(struct kmem_cache *s, struct page *page) 1504 { 1505 if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { 1506 struct rcu_head *head; 1507 1508 if (need_reserve_slab_rcu) { 1509 int order = compound_order(page); 1510 int offset = (PAGE_SIZE << order) - s->reserved; 1511 1512 VM_BUG_ON(s->reserved != sizeof(*head)); 1513 head = page_address(page) + offset; 1514 } else { 1515 /* 1516 * RCU free overloads the RCU head over the LRU 1517 */ 1518 head = (void *)&page->lru; 1519 } 1520 1521 call_rcu(head, rcu_free_slab); 1522 } else 1523 __free_slab(s, page); 1524 } 1525 1526 static void discard_slab(struct kmem_cache *s, struct page *page) 1527 { 1528 dec_slabs_node(s, page_to_nid(page), page->objects); 1529 free_slab(s, page); 1530 } 1531 1532 /* 1533 * Management of partially allocated slabs. 1534 */ 1535 static inline void 1536 __add_partial(struct kmem_cache_node *n, struct page *page, int tail) 1537 { 1538 n->nr_partial++; 1539 if (tail == DEACTIVATE_TO_TAIL) 1540 list_add_tail(&page->lru, &n->partial); 1541 else 1542 list_add(&page->lru, &n->partial); 1543 } 1544 1545 static inline void add_partial(struct kmem_cache_node *n, 1546 struct page *page, int tail) 1547 { 1548 lockdep_assert_held(&n->list_lock); 1549 __add_partial(n, page, tail); 1550 } 1551 1552 static inline void 1553 __remove_partial(struct kmem_cache_node *n, struct page *page) 1554 { 1555 list_del(&page->lru); 1556 n->nr_partial--; 1557 } 1558 1559 static inline void remove_partial(struct kmem_cache_node *n, 1560 struct page *page) 1561 { 1562 lockdep_assert_held(&n->list_lock); 1563 __remove_partial(n, page); 1564 } 1565 1566 /* 1567 * Remove slab from the partial list, freeze it and 1568 * return the pointer to the freelist. 1569 * 1570 * Returns a list of objects or NULL if it fails. 1571 */ 1572 static inline void *acquire_slab(struct kmem_cache *s, 1573 struct kmem_cache_node *n, struct page *page, 1574 int mode, int *objects) 1575 { 1576 void *freelist; 1577 unsigned long counters; 1578 struct page new; 1579 1580 lockdep_assert_held(&n->list_lock); 1581 1582 /* 1583 * Zap the freelist and set the frozen bit. 1584 * The old freelist is the list of objects for the 1585 * per cpu allocation list. 1586 */ 1587 freelist = page->freelist; 1588 counters = page->counters; 1589 new.counters = counters; 1590 *objects = new.objects - new.inuse; 1591 if (mode) { 1592 new.inuse = page->objects; 1593 new.freelist = NULL; 1594 } else { 1595 new.freelist = freelist; 1596 } 1597 1598 VM_BUG_ON(new.frozen); 1599 new.frozen = 1; 1600 1601 if (!__cmpxchg_double_slab(s, page, 1602 freelist, counters, 1603 new.freelist, new.counters, 1604 "acquire_slab")) 1605 return NULL; 1606 1607 remove_partial(n, page); 1608 WARN_ON(!freelist); 1609 return freelist; 1610 } 1611 1612 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); 1613 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); 1614 1615 /* 1616 * Try to allocate a partial slab from a specific node. 1617 */ 1618 static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, 1619 struct kmem_cache_cpu *c, gfp_t flags) 1620 { 1621 struct page *page, *page2; 1622 void *object = NULL; 1623 int available = 0; 1624 int objects; 1625 1626 /* 1627 * Racy check. If we mistakenly see no partial slabs then we 1628 * just allocate an empty slab. If we mistakenly try to get a 1629 * partial slab and there is none available then get_partials() 1630 * will return NULL. 1631 */ 1632 if (!n || !n->nr_partial) 1633 return NULL; 1634 1635 spin_lock(&n->list_lock); 1636 list_for_each_entry_safe(page, page2, &n->partial, lru) { 1637 void *t; 1638 1639 if (!pfmemalloc_match(page, flags)) 1640 continue; 1641 1642 t = acquire_slab(s, n, page, object == NULL, &objects); 1643 if (!t) 1644 break; 1645 1646 available += objects; 1647 if (!object) { 1648 c->page = page; 1649 stat(s, ALLOC_FROM_PARTIAL); 1650 object = t; 1651 } else { 1652 put_cpu_partial(s, page, 0); 1653 stat(s, CPU_PARTIAL_NODE); 1654 } 1655 if (!kmem_cache_has_cpu_partial(s) 1656 || available > s->cpu_partial / 2) 1657 break; 1658 1659 } 1660 spin_unlock(&n->list_lock); 1661 return object; 1662 } 1663 1664 /* 1665 * Get a page from somewhere. Search in increasing NUMA distances. 1666 */ 1667 static void *get_any_partial(struct kmem_cache *s, gfp_t flags, 1668 struct kmem_cache_cpu *c) 1669 { 1670 #ifdef CONFIG_NUMA 1671 struct zonelist *zonelist; 1672 struct zoneref *z; 1673 struct zone *zone; 1674 enum zone_type high_zoneidx = gfp_zone(flags); 1675 void *object; 1676 unsigned int cpuset_mems_cookie; 1677 1678 /* 1679 * The defrag ratio allows a configuration of the tradeoffs between 1680 * inter node defragmentation and node local allocations. A lower 1681 * defrag_ratio increases the tendency to do local allocations 1682 * instead of attempting to obtain partial slabs from other nodes. 1683 * 1684 * If the defrag_ratio is set to 0 then kmalloc() always 1685 * returns node local objects. If the ratio is higher then kmalloc() 1686 * may return off node objects because partial slabs are obtained 1687 * from other nodes and filled up. 1688 * 1689 * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes 1690 * defrag_ratio = 1000) then every (well almost) allocation will 1691 * first attempt to defrag slab caches on other nodes. This means 1692 * scanning over all nodes to look for partial slabs which may be 1693 * expensive if we do it every time we are trying to find a slab 1694 * with available objects. 1695 */ 1696 if (!s->remote_node_defrag_ratio || 1697 get_cycles() % 1024 > s->remote_node_defrag_ratio) 1698 return NULL; 1699 1700 do { 1701 cpuset_mems_cookie = read_mems_allowed_begin(); 1702 zonelist = node_zonelist(mempolicy_slab_node(), flags); 1703 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { 1704 struct kmem_cache_node *n; 1705 1706 n = get_node(s, zone_to_nid(zone)); 1707 1708 if (n && cpuset_zone_allowed(zone, flags) && 1709 n->nr_partial > s->min_partial) { 1710 object = get_partial_node(s, n, c, flags); 1711 if (object) { 1712 /* 1713 * Don't check read_mems_allowed_retry() 1714 * here - if mems_allowed was updated in 1715 * parallel, that was a harmless race 1716 * between allocation and the cpuset 1717 * update 1718 */ 1719 return object; 1720 } 1721 } 1722 } 1723 } while (read_mems_allowed_retry(cpuset_mems_cookie)); 1724 #endif 1725 return NULL; 1726 } 1727 1728 /* 1729 * Get a partial page, lock it and return it. 1730 */ 1731 static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, 1732 struct kmem_cache_cpu *c) 1733 { 1734 void *object; 1735 int searchnode = node; 1736 1737 if (node == NUMA_NO_NODE) 1738 searchnode = numa_mem_id(); 1739 else if (!node_present_pages(node)) 1740 searchnode = node_to_mem_node(node); 1741 1742 object = get_partial_node(s, get_node(s, searchnode), c, flags); 1743 if (object || node != NUMA_NO_NODE) 1744 return object; 1745 1746 return get_any_partial(s, flags, c); 1747 } 1748 1749 #ifdef CONFIG_PREEMPT 1750 /* 1751 * Calculate the next globally unique transaction for disambiguiation 1752 * during cmpxchg. The transactions start with the cpu number and are then 1753 * incremented by CONFIG_NR_CPUS. 1754 */ 1755 #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) 1756 #else 1757 /* 1758 * No preemption supported therefore also no need to check for 1759 * different cpus. 1760 */ 1761 #define TID_STEP 1 1762 #endif 1763 1764 static inline unsigned long next_tid(unsigned long tid) 1765 { 1766 return tid + TID_STEP; 1767 } 1768 1769 static inline unsigned int tid_to_cpu(unsigned long tid) 1770 { 1771 return tid % TID_STEP; 1772 } 1773 1774 static inline unsigned long tid_to_event(unsigned long tid) 1775 { 1776 return tid / TID_STEP; 1777 } 1778 1779 static inline unsigned int init_tid(int cpu) 1780 { 1781 return cpu; 1782 } 1783 1784 static inline void note_cmpxchg_failure(const char *n, 1785 const struct kmem_cache *s, unsigned long tid) 1786 { 1787 #ifdef SLUB_DEBUG_CMPXCHG 1788 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); 1789 1790 pr_info("%s %s: cmpxchg redo ", n, s->name); 1791 1792 #ifdef CONFIG_PREEMPT 1793 if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) 1794 pr_warn("due to cpu change %d -> %d\n", 1795 tid_to_cpu(tid), tid_to_cpu(actual_tid)); 1796 else 1797 #endif 1798 if (tid_to_event(tid) != tid_to_event(actual_tid)) 1799 pr_warn("due to cpu running other code. Event %ld->%ld\n", 1800 tid_to_event(tid), tid_to_event(actual_tid)); 1801 else 1802 pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", 1803 actual_tid, tid, next_tid(tid)); 1804 #endif 1805 stat(s, CMPXCHG_DOUBLE_CPU_FAIL); 1806 } 1807 1808 static void init_kmem_cache_cpus(struct kmem_cache *s) 1809 { 1810 int cpu; 1811 1812 for_each_possible_cpu(cpu) 1813 per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); 1814 } 1815 1816 /* 1817 * Remove the cpu slab 1818 */ 1819 static void deactivate_slab(struct kmem_cache *s, struct page *page, 1820 void *freelist) 1821 { 1822 enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; 1823 struct kmem_cache_node *n = get_node(s, page_to_nid(page)); 1824 int lock = 0; 1825 enum slab_modes l = M_NONE, m = M_NONE; 1826 void *nextfree; 1827 int tail = DEACTIVATE_TO_HEAD; 1828 struct page new; 1829 struct page old; 1830 1831 if (page->freelist) { 1832 stat(s, DEACTIVATE_REMOTE_FREES); 1833 tail = DEACTIVATE_TO_TAIL; 1834 } 1835 1836 /* 1837 * Stage one: Free all available per cpu objects back 1838 * to the page freelist while it is still frozen. Leave the 1839 * last one. 1840 * 1841 * There is no need to take the list->lock because the page 1842 * is still frozen. 1843 */ 1844 while (freelist && (nextfree = get_freepointer(s, freelist))) { 1845 void *prior; 1846 unsigned long counters; 1847 1848 do { 1849 prior = page->freelist; 1850 counters = page->counters; 1851 set_freepointer(s, freelist, prior); 1852 new.counters = counters; 1853 new.inuse--; 1854 VM_BUG_ON(!new.frozen); 1855 1856 } while (!__cmpxchg_double_slab(s, page, 1857 prior, counters, 1858 freelist, new.counters, 1859 "drain percpu freelist")); 1860 1861 freelist = nextfree; 1862 } 1863 1864 /* 1865 * Stage two: Ensure that the page is unfrozen while the 1866 * list presence reflects the actual number of objects 1867 * during unfreeze. 1868 * 1869 * We setup the list membership and then perform a cmpxchg 1870 * with the count. If there is a mismatch then the page 1871 * is not unfrozen but the page is on the wrong list. 1872 * 1873 * Then we restart the process which may have to remove 1874 * the page from the list that we just put it on again 1875 * because the number of objects in the slab may have 1876 * changed. 1877 */ 1878 redo: 1879 1880 old.freelist = page->freelist; 1881 old.counters = page->counters; 1882 VM_BUG_ON(!old.frozen); 1883 1884 /* Determine target state of the slab */ 1885 new.counters = old.counters; 1886 if (freelist) { 1887 new.inuse--; 1888 set_freepointer(s, freelist, old.freelist); 1889 new.freelist = freelist; 1890 } else 1891 new.freelist = old.freelist; 1892 1893 new.frozen = 0; 1894 1895 if (!new.inuse && n->nr_partial >= s->min_partial) 1896 m = M_FREE; 1897 else if (new.freelist) { 1898 m = M_PARTIAL; 1899 if (!lock) { 1900 lock = 1; 1901 /* 1902 * Taking the spinlock removes the possiblity 1903 * that acquire_slab() will see a slab page that 1904 * is frozen 1905 */ 1906 spin_lock(&n->list_lock); 1907 } 1908 } else { 1909 m = M_FULL; 1910 if (kmem_cache_debug(s) && !lock) { 1911 lock = 1; 1912 /* 1913 * This also ensures that the scanning of full 1914 * slabs from diagnostic functions will not see 1915 * any frozen slabs. 1916 */ 1917 spin_lock(&n->list_lock); 1918 } 1919 } 1920 1921 if (l != m) { 1922 1923 if (l == M_PARTIAL) 1924 1925 remove_partial(n, page); 1926 1927 else if (l == M_FULL) 1928 1929 remove_full(s, n, page); 1930 1931 if (m == M_PARTIAL) { 1932 1933 add_partial(n, page, tail); 1934 stat(s, tail); 1935 1936 } else if (m == M_FULL) { 1937 1938 stat(s, DEACTIVATE_FULL); 1939 add_full(s, n, page); 1940 1941 } 1942 } 1943 1944 l = m; 1945 if (!__cmpxchg_double_slab(s, page, 1946 old.freelist, old.counters, 1947 new.freelist, new.counters, 1948 "unfreezing slab")) 1949 goto redo; 1950 1951 if (lock) 1952 spin_unlock(&n->list_lock); 1953 1954 if (m == M_FREE) { 1955 stat(s, DEACTIVATE_EMPTY); 1956 discard_slab(s, page); 1957 stat(s, FREE_SLAB); 1958 } 1959 } 1960 1961 /* 1962 * Unfreeze all the cpu partial slabs. 1963 * 1964 * This function must be called with interrupts disabled 1965 * for the cpu using c (or some other guarantee must be there 1966 * to guarantee no concurrent accesses). 1967 */ 1968 static void unfreeze_partials(struct kmem_cache *s, 1969 struct kmem_cache_cpu *c) 1970 { 1971 #ifdef CONFIG_SLUB_CPU_PARTIAL 1972 struct kmem_cache_node *n = NULL, *n2 = NULL; 1973 struct page *page, *discard_page = NULL; 1974 1975 while ((page = c->partial)) { 1976 struct page new; 1977 struct page old; 1978 1979 c->partial = page->next; 1980 1981 n2 = get_node(s, page_to_nid(page)); 1982 if (n != n2) { 1983 if (n) 1984 spin_unlock(&n->list_lock); 1985 1986 n = n2; 1987 spin_lock(&n->list_lock); 1988 } 1989 1990 do { 1991 1992 old.freelist = page->freelist; 1993 old.counters = page->counters; 1994 VM_BUG_ON(!old.frozen); 1995 1996 new.counters = old.counters; 1997 new.freelist = old.freelist; 1998 1999 new.frozen = 0; 2000 2001 } while (!__cmpxchg_double_slab(s, page, 2002 old.freelist, old.counters, 2003 new.freelist, new.counters, 2004 "unfreezing slab")); 2005 2006 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { 2007 page->next = discard_page; 2008 discard_page = page; 2009 } else { 2010 add_partial(n, page, DEACTIVATE_TO_TAIL); 2011 stat(s, FREE_ADD_PARTIAL); 2012 } 2013 } 2014 2015 if (n) 2016 spin_unlock(&n->list_lock); 2017 2018 while (discard_page) { 2019 page = discard_page; 2020 discard_page = discard_page->next; 2021 2022 stat(s, DEACTIVATE_EMPTY); 2023 discard_slab(s, page); 2024 stat(s, FREE_SLAB); 2025 } 2026 #endif 2027 } 2028 2029 /* 2030 * Put a page that was just frozen (in __slab_free) into a partial page 2031 * slot if available. This is done without interrupts disabled and without 2032 * preemption disabled. The cmpxchg is racy and may put the partial page 2033 * onto a random cpus partial slot. 2034 * 2035 * If we did not find a slot then simply move all the partials to the 2036 * per node partial list. 2037 */ 2038 static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) 2039 { 2040 #ifdef CONFIG_SLUB_CPU_PARTIAL 2041 struct page *oldpage; 2042 int pages; 2043 int pobjects; 2044 2045 preempt_disable(); 2046 do { 2047 pages = 0; 2048 pobjects = 0; 2049 oldpage = this_cpu_read(s->cpu_slab->partial); 2050 2051 if (oldpage) { 2052 pobjects = oldpage->pobjects; 2053 pages = oldpage->pages; 2054 if (drain && pobjects > s->cpu_partial) { 2055 unsigned long flags; 2056 /* 2057 * partial array is full. Move the existing 2058 * set to the per node partial list. 2059 */ 2060 local_irq_save(flags); 2061 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); 2062 local_irq_restore(flags); 2063 oldpage = NULL; 2064 pobjects = 0; 2065 pages = 0; 2066 stat(s, CPU_PARTIAL_DRAIN); 2067 } 2068 } 2069 2070 pages++; 2071 pobjects += page->objects - page->inuse; 2072 2073 page->pages = pages; 2074 page->pobjects = pobjects; 2075 page->next = oldpage; 2076 2077 } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) 2078 != oldpage); 2079 if (unlikely(!s->cpu_partial)) { 2080 unsigned long flags; 2081 2082 local_irq_save(flags); 2083 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); 2084 local_irq_restore(flags); 2085 } 2086 preempt_enable(); 2087 #endif 2088 } 2089 2090 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) 2091 { 2092 stat(s, CPUSLAB_FLUSH); 2093 deactivate_slab(s, c->page, c->freelist); 2094 2095 c->tid = next_tid(c->tid); 2096 c->page = NULL; 2097 c->freelist = NULL; 2098 } 2099 2100 /* 2101 * Flush cpu slab. 2102 * 2103 * Called from IPI handler with interrupts disabled. 2104 */ 2105 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) 2106 { 2107 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 2108 2109 if (likely(c)) { 2110 if (c->page) 2111 flush_slab(s, c); 2112 2113 unfreeze_partials(s, c); 2114 } 2115 } 2116 2117 static void flush_cpu_slab(void *d) 2118 { 2119 struct kmem_cache *s = d; 2120 2121 __flush_cpu_slab(s, smp_processor_id()); 2122 } 2123 2124 static bool has_cpu_slab(int cpu, void *info) 2125 { 2126 struct kmem_cache *s = info; 2127 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); 2128 2129 return c->page || c->partial; 2130 } 2131 2132 static void flush_all(struct kmem_cache *s) 2133 { 2134 on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); 2135 } 2136 2137 /* 2138 * Check if the objects in a per cpu structure fit numa 2139 * locality expectations. 2140 */ 2141 static inline int node_match(struct page *page, int node) 2142 { 2143 #ifdef CONFIG_NUMA 2144 if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node)) 2145 return 0; 2146 #endif 2147 return 1; 2148 } 2149 2150 #ifdef CONFIG_SLUB_DEBUG 2151 static int count_free(struct page *page) 2152 { 2153 return page->objects - page->inuse; 2154 } 2155 2156 static inline unsigned long node_nr_objs(struct kmem_cache_node *n) 2157 { 2158 return atomic_long_read(&n->total_objects); 2159 } 2160 #endif /* CONFIG_SLUB_DEBUG */ 2161 2162 #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) 2163 static unsigned long count_partial(struct kmem_cache_node *n, 2164 int (*get_count)(struct page *)) 2165 { 2166 unsigned long flags; 2167 unsigned long x = 0; 2168 struct page *page; 2169 2170 spin_lock_irqsave(&n->list_lock, flags); 2171 list_for_each_entry(page, &n->partial, lru) 2172 x += get_count(page); 2173 spin_unlock_irqrestore(&n->list_lock, flags); 2174 return x; 2175 } 2176 #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ 2177 2178 static noinline void 2179 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) 2180 { 2181 #ifdef CONFIG_SLUB_DEBUG 2182 static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, 2183 DEFAULT_RATELIMIT_BURST); 2184 int node; 2185 struct kmem_cache_node *n; 2186 2187 if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) 2188 return; 2189 2190 pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", 2191 nid, gfpflags); 2192 pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n", 2193 s->name, s->object_size, s->size, oo_order(s->oo), 2194 oo_order(s->min)); 2195 2196 if (oo_order(s->min) > get_order(s->object_size)) 2197 pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", 2198 s->name); 2199 2200 for_each_kmem_cache_node(s, node, n) { 2201 unsigned long nr_slabs; 2202 unsigned long nr_objs; 2203 unsigned long nr_free; 2204 2205 nr_free = count_partial(n, count_free); 2206 nr_slabs = node_nr_slabs(n); 2207 nr_objs = node_nr_objs(n); 2208 2209 pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", 2210 node, nr_slabs, nr_objs, nr_free); 2211 } 2212 #endif 2213 } 2214 2215 static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, 2216 int node, struct kmem_cache_cpu **pc) 2217 { 2218 void *freelist; 2219 struct kmem_cache_cpu *c = *pc; 2220 struct page *page; 2221 2222 freelist = get_partial(s, flags, node, c); 2223 2224 if (freelist) 2225 return freelist; 2226 2227 page = new_slab(s, flags, node); 2228 if (page) { 2229 c = raw_cpu_ptr(s->cpu_slab); 2230 if (c->page) 2231 flush_slab(s, c); 2232 2233 /* 2234 * No other reference to the page yet so we can 2235 * muck around with it freely without cmpxchg 2236 */ 2237 freelist = page->freelist; 2238 page->freelist = NULL; 2239 2240 stat(s, ALLOC_SLAB); 2241 c->page = page; 2242 *pc = c; 2243 } else 2244 freelist = NULL; 2245 2246 return freelist; 2247 } 2248 2249 static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) 2250 { 2251 if (unlikely(PageSlabPfmemalloc(page))) 2252 return gfp_pfmemalloc_allowed(gfpflags); 2253 2254 return true; 2255 } 2256 2257 /* 2258 * Check the page->freelist of a page and either transfer the freelist to the 2259 * per cpu freelist or deactivate the page. 2260 * 2261 * The page is still frozen if the return value is not NULL. 2262 * 2263 * If this function returns NULL then the page has been unfrozen. 2264 * 2265 * This function must be called with interrupt disabled. 2266 */ 2267 static inline void *get_freelist(struct kmem_cache *s, struct page *page) 2268 { 2269 struct page new; 2270 unsigned long counters; 2271 void *freelist; 2272 2273 do { 2274 freelist = page->freelist; 2275 counters = page->counters; 2276 2277 new.counters = counters; 2278 VM_BUG_ON(!new.frozen); 2279 2280 new.inuse = page->objects; 2281 new.frozen = freelist != NULL; 2282 2283 } while (!__cmpxchg_double_slab(s, page, 2284 freelist, counters, 2285 NULL, new.counters, 2286 "get_freelist")); 2287 2288 return freelist; 2289 } 2290 2291 /* 2292 * Slow path. The lockless freelist is empty or we need to perform 2293 * debugging duties. 2294 * 2295 * Processing is still very fast if new objects have been freed to the 2296 * regular freelist. In that case we simply take over the regular freelist 2297 * as the lockless freelist and zap the regular freelist. 2298 * 2299 * If that is not working then we fall back to the partial lists. We take the 2300 * first element of the freelist as the object to allocate now and move the 2301 * rest of the freelist to the lockless freelist. 2302 * 2303 * And if we were unable to get a new slab from the partial slab lists then 2304 * we need to allocate a new slab. This is the slowest path since it involves 2305 * a call to the page allocator and the setup of a new slab. 2306 */ 2307 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, 2308 unsigned long addr, struct kmem_cache_cpu *c) 2309 { 2310 void *freelist; 2311 struct page *page; 2312 unsigned long flags; 2313 2314 local_irq_save(flags); 2315 #ifdef CONFIG_PREEMPT 2316 /* 2317 * We may have been preempted and rescheduled on a different 2318 * cpu before disabling interrupts. Need to reload cpu area 2319 * pointer. 2320 */ 2321 c = this_cpu_ptr(s->cpu_slab); 2322 #endif 2323 2324 page = c->page; 2325 if (!page) 2326 goto new_slab; 2327 redo: 2328 2329 if (unlikely(!node_match(page, node))) { 2330 int searchnode = node; 2331 2332 if (node != NUMA_NO_NODE && !node_present_pages(node)) 2333 searchnode = node_to_mem_node(node); 2334 2335 if (unlikely(!node_match(page, searchnode))) { 2336 stat(s, ALLOC_NODE_MISMATCH); 2337 deactivate_slab(s, page, c->freelist); 2338 c->page = NULL; 2339 c->freelist = NULL; 2340 goto new_slab; 2341 } 2342 } 2343 2344 /* 2345 * By rights, we should be searching for a slab page that was 2346 * PFMEMALLOC but right now, we are losing the pfmemalloc 2347 * information when the page leaves the per-cpu allocator 2348 */ 2349 if (unlikely(!pfmemalloc_match(page, gfpflags))) { 2350 deactivate_slab(s, page, c->freelist); 2351 c->page = NULL; 2352 c->freelist = NULL; 2353 goto new_slab; 2354 } 2355 2356 /* must check again c->freelist in case of cpu migration or IRQ */ 2357 freelist = c->freelist; 2358 if (freelist) 2359 goto load_freelist; 2360 2361 freelist = get_freelist(s, page); 2362 2363 if (!freelist) { 2364 c->page = NULL; 2365 stat(s, DEACTIVATE_BYPASS); 2366 goto new_slab; 2367 } 2368 2369 stat(s, ALLOC_REFILL); 2370 2371 load_freelist: 2372 /* 2373 * freelist is pointing to the list of objects to be used. 2374 * page is pointing to the page from which the objects are obtained. 2375 * That page must be frozen for per cpu allocations to work. 2376 */ 2377 VM_BUG_ON(!c->page->frozen); 2378 c->freelist = get_freepointer(s, freelist); 2379 c->tid = next_tid(c->tid); 2380 local_irq_restore(flags); 2381 return freelist; 2382 2383 new_slab: 2384 2385 if (c->partial) { 2386 page = c->page = c->partial; 2387 c->partial = page->next; 2388 stat(s, CPU_PARTIAL_ALLOC); 2389 c->freelist = NULL; 2390 goto redo; 2391 } 2392 2393 freelist = new_slab_objects(s, gfpflags, node, &c); 2394 2395 if (unlikely(!freelist)) { 2396 slab_out_of_memory(s, gfpflags, node); 2397 local_irq_restore(flags); 2398 return NULL; 2399 } 2400 2401 page = c->page; 2402 if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) 2403 goto load_freelist; 2404 2405 /* Only entered in the debug case */ 2406 if (kmem_cache_debug(s) && 2407 !alloc_debug_processing(s, page, freelist, addr)) 2408 goto new_slab; /* Slab failed checks. Next slab needed */ 2409 2410 deactivate_slab(s, page, get_freepointer(s, freelist)); 2411 c->page = NULL; 2412 c->freelist = NULL; 2413 local_irq_restore(flags); 2414 return freelist; 2415 } 2416 2417 /* 2418 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) 2419 * have the fastpath folded into their functions. So no function call 2420 * overhead for requests that can be satisfied on the fastpath. 2421 * 2422 * The fastpath works by first checking if the lockless freelist can be used. 2423 * If not then __slab_alloc is called for slow processing. 2424 * 2425 * Otherwise we can simply pick the next object from the lockless free list. 2426 */ 2427 static __always_inline void *slab_alloc_node(struct kmem_cache *s, 2428 gfp_t gfpflags, int node, unsigned long addr) 2429 { 2430 void **object; 2431 struct kmem_cache_cpu *c; 2432 struct page *page; 2433 unsigned long tid; 2434 2435 s = slab_pre_alloc_hook(s, gfpflags); 2436 if (!s) 2437 return NULL; 2438 redo: 2439 /* 2440 * Must read kmem_cache cpu data via this cpu ptr. Preemption is 2441 * enabled. We may switch back and forth between cpus while 2442 * reading from one cpu area. That does not matter as long 2443 * as we end up on the original cpu again when doing the cmpxchg. 2444 * 2445 * We should guarantee that tid and kmem_cache are retrieved on 2446 * the same cpu. It could be different if CONFIG_PREEMPT so we need 2447 * to check if it is matched or not. 2448 */ 2449 do { 2450 tid = this_cpu_read(s->cpu_slab->tid); 2451 c = raw_cpu_ptr(s->cpu_slab); 2452 } while (IS_ENABLED(CONFIG_PREEMPT) && unlikely(tid != c->tid)); 2453 2454 /* 2455 * Irqless object alloc/free algorithm used here depends on sequence 2456 * of fetching cpu_slab's data. tid should be fetched before anything 2457 * on c to guarantee that object and page associated with previous tid 2458 * won't be used with current tid. If we fetch tid first, object and 2459 * page could be one associated with next tid and our alloc/free 2460 * request will be failed. In this case, we will retry. So, no problem. 2461 */ 2462 barrier(); 2463 2464 /* 2465 * The transaction ids are globally unique per cpu and per operation on 2466 * a per cpu queue. Thus they can be guarantee that the cmpxchg_double 2467 * occurs on the right processor and that there was no operation on the 2468 * linked list in between. 2469 */ 2470 2471 object = c->freelist; 2472 page = c->page; 2473 if (unlikely(!object || !node_match(page, node))) { 2474 object = __slab_alloc(s, gfpflags, node, addr, c); 2475 stat(s, ALLOC_SLOWPATH); 2476 } else { 2477 void *next_object = get_freepointer_safe(s, object); 2478 2479 /* 2480 * The cmpxchg will only match if there was no additional 2481 * operation and if we are on the right processor. 2482 * 2483 * The cmpxchg does the following atomically (without lock 2484 * semantics!) 2485 * 1. Relocate first pointer to the current per cpu area. 2486 * 2. Verify that tid and freelist have not been changed 2487 * 3. If they were not changed replace tid and freelist 2488 * 2489 * Since this is without lock semantics the protection is only 2490 * against code executing on this cpu *not* from access by 2491 * other cpus. 2492 */ 2493 if (unlikely(!this_cpu_cmpxchg_double( 2494 s->cpu_slab->freelist, s->cpu_slab->tid, 2495 object, tid, 2496 next_object, next_tid(tid)))) { 2497 2498 note_cmpxchg_failure("slab_alloc", s, tid); 2499 goto redo; 2500 } 2501 prefetch_freepointer(s, next_object); 2502 stat(s, ALLOC_FASTPATH); 2503 } 2504 2505 if (unlikely(gfpflags & __GFP_ZERO) && object) 2506 memset(object, 0, s->object_size); 2507 2508 slab_post_alloc_hook(s, gfpflags, object); 2509 2510 return object; 2511 } 2512 2513 static __always_inline void *slab_alloc(struct kmem_cache *s, 2514 gfp_t gfpflags, unsigned long addr) 2515 { 2516 return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); 2517 } 2518 2519 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 2520 { 2521 void *ret = slab_alloc(s, gfpflags, _RET_IP_); 2522 2523 trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, 2524 s->size, gfpflags); 2525 2526 return ret; 2527 } 2528 EXPORT_SYMBOL(kmem_cache_alloc); 2529 2530 #ifdef CONFIG_TRACING 2531 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) 2532 { 2533 void *ret = slab_alloc(s, gfpflags, _RET_IP_); 2534 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); 2535 kasan_kmalloc(s, ret, size); 2536 return ret; 2537 } 2538 EXPORT_SYMBOL(kmem_cache_alloc_trace); 2539 #endif 2540 2541 #ifdef CONFIG_NUMA 2542 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 2543 { 2544 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); 2545 2546 trace_kmem_cache_alloc_node(_RET_IP_, ret, 2547 s->object_size, s->size, gfpflags, node); 2548 2549 return ret; 2550 } 2551 EXPORT_SYMBOL(kmem_cache_alloc_node); 2552 2553 #ifdef CONFIG_TRACING 2554 void *kmem_cache_alloc_node_trace(struct kmem_cache *s, 2555 gfp_t gfpflags, 2556 int node, size_t size) 2557 { 2558 void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); 2559 2560 trace_kmalloc_node(_RET_IP_, ret, 2561 size, s->size, gfpflags, node); 2562 2563 kasan_kmalloc(s, ret, size); 2564 return ret; 2565 } 2566 EXPORT_SYMBOL(kmem_cache_alloc_node_trace); 2567 #endif 2568 #endif 2569 2570 /* 2571 * Slow path handling. This may still be called frequently since objects 2572 * have a longer lifetime than the cpu slabs in most processing loads. 2573 * 2574 * So we still attempt to reduce cache line usage. Just take the slab 2575 * lock and free the item. If there is no additional partial page 2576 * handling required then we can return immediately. 2577 */ 2578 static void __slab_free(struct kmem_cache *s, struct page *page, 2579 void *x, unsigned long addr) 2580 { 2581 void *prior; 2582 void **object = (void *)x; 2583 int was_frozen; 2584 struct page new; 2585 unsigned long counters; 2586 struct kmem_cache_node *n = NULL; 2587 unsigned long uninitialized_var(flags); 2588 2589 stat(s, FREE_SLOWPATH); 2590 2591 if (kmem_cache_debug(s) && 2592 !(n = free_debug_processing(s, page, x, addr, &flags))) 2593 return; 2594 2595 do { 2596 if (unlikely(n)) { 2597 spin_unlock_irqrestore(&n->list_lock, flags); 2598 n = NULL; 2599 } 2600 prior = page->freelist; 2601 counters = page->counters; 2602 set_freepointer(s, object, prior); 2603 new.counters = counters; 2604 was_frozen = new.frozen; 2605 new.inuse--; 2606 if ((!new.inuse || !prior) && !was_frozen) { 2607 2608 if (kmem_cache_has_cpu_partial(s) && !prior) { 2609 2610 /* 2611 * Slab was on no list before and will be 2612 * partially empty 2613 * We can defer the list move and instead 2614 * freeze it. 2615 */ 2616 new.frozen = 1; 2617 2618 } else { /* Needs to be taken off a list */ 2619 2620 n = get_node(s, page_to_nid(page)); 2621 /* 2622 * Speculatively acquire the list_lock. 2623 * If the cmpxchg does not succeed then we may 2624 * drop the list_lock without any processing. 2625 * 2626 * Otherwise the list_lock will synchronize with 2627 * other processors updating the list of slabs. 2628 */ 2629 spin_lock_irqsave(&n->list_lock, flags); 2630 2631 } 2632 } 2633 2634 } while (!cmpxchg_double_slab(s, page, 2635 prior, counters, 2636 object, new.counters, 2637 "__slab_free")); 2638 2639 if (likely(!n)) { 2640 2641 /* 2642 * If we just froze the page then put it onto the 2643 * per cpu partial list. 2644 */ 2645 if (new.frozen && !was_frozen) { 2646 put_cpu_partial(s, page, 1); 2647 stat(s, CPU_PARTIAL_FREE); 2648 } 2649 /* 2650 * The list lock was not taken therefore no list 2651 * activity can be necessary. 2652 */ 2653 if (was_frozen) 2654 stat(s, FREE_FROZEN); 2655 return; 2656 } 2657 2658 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) 2659 goto slab_empty; 2660 2661 /* 2662 * Objects left in the slab. If it was not on the partial list before 2663 * then add it. 2664 */ 2665 if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { 2666 if (kmem_cache_debug(s)) 2667 remove_full(s, n, page); 2668 add_partial(n, page, DEACTIVATE_TO_TAIL); 2669 stat(s, FREE_ADD_PARTIAL); 2670 } 2671 spin_unlock_irqrestore(&n->list_lock, flags); 2672 return; 2673 2674 slab_empty: 2675 if (prior) { 2676 /* 2677 * Slab on the partial list. 2678 */ 2679 remove_partial(n, page); 2680 stat(s, FREE_REMOVE_PARTIAL); 2681 } else { 2682 /* Slab must be on the full list */ 2683 remove_full(s, n, page); 2684 } 2685 2686 spin_unlock_irqrestore(&n->list_lock, flags); 2687 stat(s, FREE_SLAB); 2688 discard_slab(s, page); 2689 } 2690 2691 /* 2692 * Fastpath with forced inlining to produce a kfree and kmem_cache_free that 2693 * can perform fastpath freeing without additional function calls. 2694 * 2695 * The fastpath is only possible if we are freeing to the current cpu slab 2696 * of this processor. This typically the case if we have just allocated 2697 * the item before. 2698 * 2699 * If fastpath is not possible then fall back to __slab_free where we deal 2700 * with all sorts of special processing. 2701 */ 2702 static __always_inline void slab_free(struct kmem_cache *s, 2703 struct page *page, void *x, unsigned long addr) 2704 { 2705 void **object = (void *)x; 2706 struct kmem_cache_cpu *c; 2707 unsigned long tid; 2708 2709 slab_free_hook(s, x); 2710 2711 redo: 2712 /* 2713 * Determine the currently cpus per cpu slab. 2714 * The cpu may change afterward. However that does not matter since 2715 * data is retrieved via this pointer. If we are on the same cpu 2716 * during the cmpxchg then the free will succedd. 2717 */ 2718 do { 2719 tid = this_cpu_read(s->cpu_slab->tid); 2720 c = raw_cpu_ptr(s->cpu_slab); 2721 } while (IS_ENABLED(CONFIG_PREEMPT) && unlikely(tid != c->tid)); 2722 2723 /* Same with comment on barrier() in slab_alloc_node() */ 2724 barrier(); 2725 2726 if (likely(page == c->page)) { 2727 set_freepointer(s, object, c->freelist); 2728 2729 if (unlikely(!this_cpu_cmpxchg_double( 2730 s->cpu_slab->freelist, s->cpu_slab->tid, 2731 c->freelist, tid, 2732 object, next_tid(tid)))) { 2733 2734 note_cmpxchg_failure("slab_free", s, tid); 2735 goto redo; 2736 } 2737 stat(s, FREE_FASTPATH); 2738 } else 2739 __slab_free(s, page, x, addr); 2740 2741 } 2742 2743 void kmem_cache_free(struct kmem_cache *s, void *x) 2744 { 2745 s = cache_from_obj(s, x); 2746 if (!s) 2747 return; 2748 slab_free(s, virt_to_head_page(x), x, _RET_IP_); 2749 trace_kmem_cache_free(_RET_IP_, x); 2750 } 2751 EXPORT_SYMBOL(kmem_cache_free); 2752 2753 /* 2754 * Object placement in a slab is made very easy because we always start at 2755 * offset 0. If we tune the size of the object to the alignment then we can 2756 * get the required alignment by putting one properly sized object after 2757 * another. 2758 * 2759 * Notice that the allocation order determines the sizes of the per cpu 2760 * caches. Each processor has always one slab available for allocations. 2761 * Increasing the allocation order reduces the number of times that slabs 2762 * must be moved on and off the partial lists and is therefore a factor in 2763 * locking overhead. 2764 */ 2765 2766 /* 2767 * Mininum / Maximum order of slab pages. This influences locking overhead 2768 * and slab fragmentation. A higher order reduces the number of partial slabs 2769 * and increases the number of allocations possible without having to 2770 * take the list_lock. 2771 */ 2772 static int slub_min_order; 2773 static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; 2774 static int slub_min_objects; 2775 2776 /* 2777 * Calculate the order of allocation given an slab object size. 2778 * 2779 * The order of allocation has significant impact on performance and other 2780 * system components. Generally order 0 allocations should be preferred since 2781 * order 0 does not cause fragmentation in the page allocator. Larger objects 2782 * be problematic to put into order 0 slabs because there may be too much 2783 * unused space left. We go to a higher order if more than 1/16th of the slab 2784 * would be wasted. 2785 * 2786 * In order to reach satisfactory performance we must ensure that a minimum 2787 * number of objects is in one slab. Otherwise we may generate too much 2788 * activity on the partial lists which requires taking the list_lock. This is 2789 * less a concern for large slabs though which are rarely used. 2790 * 2791 * slub_max_order specifies the order where we begin to stop considering the 2792 * number of objects in a slab as critical. If we reach slub_max_order then 2793 * we try to keep the page order as low as possible. So we accept more waste 2794 * of space in favor of a small page order. 2795 * 2796 * Higher order allocations also allow the placement of more objects in a 2797 * slab and thereby reduce object handling overhead. If the user has 2798 * requested a higher mininum order then we start with that one instead of 2799 * the smallest order which will fit the object. 2800 */ 2801 static inline int slab_order(int size, int min_objects, 2802 int max_order, int fract_leftover, int reserved) 2803 { 2804 int order; 2805 int rem; 2806 int min_order = slub_min_order; 2807 2808 if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE) 2809 return get_order(size * MAX_OBJS_PER_PAGE) - 1; 2810 2811 for (order = max(min_order, 2812 fls(min_objects * size - 1) - PAGE_SHIFT); 2813 order <= max_order; order++) { 2814 2815 unsigned long slab_size = PAGE_SIZE << order; 2816 2817 if (slab_size < min_objects * size + reserved) 2818 continue; 2819 2820 rem = (slab_size - reserved) % size; 2821 2822 if (rem <= slab_size / fract_leftover) 2823 break; 2824 2825 } 2826 2827 return order; 2828 } 2829 2830 static inline int calculate_order(int size, int reserved) 2831 { 2832 int order; 2833 int min_objects; 2834 int fraction; 2835 int max_objects; 2836 2837 /* 2838 * Attempt to find best configuration for a slab. This 2839 * works by first attempting to generate a layout with 2840 * the best configuration and backing off gradually. 2841 * 2842 * First we reduce the acceptable waste in a slab. Then 2843 * we reduce the minimum objects required in a slab. 2844 */ 2845 min_objects = slub_min_objects; 2846 if (!min_objects) 2847 min_objects = 4 * (fls(nr_cpu_ids) + 1); 2848 max_objects = order_objects(slub_max_order, size, reserved); 2849 min_objects = min(min_objects, max_objects); 2850 2851 while (min_objects > 1) { 2852 fraction = 16; 2853 while (fraction >= 4) { 2854 order = slab_order(size, min_objects, 2855 slub_max_order, fraction, reserved); 2856 if (order <= slub_max_order) 2857 return order; 2858 fraction /= 2; 2859 } 2860 min_objects--; 2861 } 2862 2863 /* 2864 * We were unable to place multiple objects in a slab. Now 2865 * lets see if we can place a single object there. 2866 */ 2867 order = slab_order(size, 1, slub_max_order, 1, reserved); 2868 if (order <= slub_max_order) 2869 return order; 2870 2871 /* 2872 * Doh this slab cannot be placed using slub_max_order. 2873 */ 2874 order = slab_order(size, 1, MAX_ORDER, 1, reserved); 2875 if (order < MAX_ORDER) 2876 return order; 2877 return -ENOSYS; 2878 } 2879 2880 static void 2881 init_kmem_cache_node(struct kmem_cache_node *n) 2882 { 2883 n->nr_partial = 0; 2884 spin_lock_init(&n->list_lock); 2885 INIT_LIST_HEAD(&n->partial); 2886 #ifdef CONFIG_SLUB_DEBUG 2887 atomic_long_set(&n->nr_slabs, 0); 2888 atomic_long_set(&n->total_objects, 0); 2889 INIT_LIST_HEAD(&n->full); 2890 #endif 2891 } 2892 2893 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) 2894 { 2895 BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < 2896 KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu)); 2897 2898 /* 2899 * Must align to double word boundary for the double cmpxchg 2900 * instructions to work; see __pcpu_double_call_return_bool(). 2901 */ 2902 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), 2903 2 * sizeof(void *)); 2904 2905 if (!s->cpu_slab) 2906 return 0; 2907 2908 init_kmem_cache_cpus(s); 2909 2910 return 1; 2911 } 2912 2913 static struct kmem_cache *kmem_cache_node; 2914 2915 /* 2916 * No kmalloc_node yet so do it by hand. We know that this is the first 2917 * slab on the node for this slabcache. There are no concurrent accesses 2918 * possible. 2919 * 2920 * Note that this function only works on the kmem_cache_node 2921 * when allocating for the kmem_cache_node. This is used for bootstrapping 2922 * memory on a fresh node that has no slab structures yet. 2923 */ 2924 static void early_kmem_cache_node_alloc(int node) 2925 { 2926 struct page *page; 2927 struct kmem_cache_node *n; 2928 2929 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); 2930 2931 page = new_slab(kmem_cache_node, GFP_NOWAIT, node); 2932 2933 BUG_ON(!page); 2934 if (page_to_nid(page) != node) { 2935 pr_err("SLUB: Unable to allocate memory from node %d\n", node); 2936 pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); 2937 } 2938 2939 n = page->freelist; 2940 BUG_ON(!n); 2941 page->freelist = get_freepointer(kmem_cache_node, n); 2942 page->inuse = 1; 2943 page->frozen = 0; 2944 kmem_cache_node->node[node] = n; 2945 #ifdef CONFIG_SLUB_DEBUG 2946 init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); 2947 init_tracking(kmem_cache_node, n); 2948 #endif 2949 kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); 2950 init_kmem_cache_node(n); 2951 inc_slabs_node(kmem_cache_node, node, page->objects); 2952 2953 /* 2954 * No locks need to be taken here as it has just been 2955 * initialized and there is no concurrent access. 2956 */ 2957 __add_partial(n, page, DEACTIVATE_TO_HEAD); 2958 } 2959 2960 static void free_kmem_cache_nodes(struct kmem_cache *s) 2961 { 2962 int node; 2963 struct kmem_cache_node *n; 2964 2965 for_each_kmem_cache_node(s, node, n) { 2966 kmem_cache_free(kmem_cache_node, n); 2967 s->node[node] = NULL; 2968 } 2969 } 2970 2971 static int init_kmem_cache_nodes(struct kmem_cache *s) 2972 { 2973 int node; 2974 2975 for_each_node_state(node, N_NORMAL_MEMORY) { 2976 struct kmem_cache_node *n; 2977 2978 if (slab_state == DOWN) { 2979 early_kmem_cache_node_alloc(node); 2980 continue; 2981 } 2982 n = kmem_cache_alloc_node(kmem_cache_node, 2983 GFP_KERNEL, node); 2984 2985 if (!n) { 2986 free_kmem_cache_nodes(s); 2987 return 0; 2988 } 2989 2990 s->node[node] = n; 2991 init_kmem_cache_node(n); 2992 } 2993 return 1; 2994 } 2995 2996 static void set_min_partial(struct kmem_cache *s, unsigned long min) 2997 { 2998 if (min < MIN_PARTIAL) 2999 min = MIN_PARTIAL; 3000 else if (min > MAX_PARTIAL) 3001 min = MAX_PARTIAL; 3002 s->min_partial = min; 3003 } 3004 3005 /* 3006 * calculate_sizes() determines the order and the distribution of data within 3007 * a slab object. 3008 */ 3009 static int calculate_sizes(struct kmem_cache *s, int forced_order) 3010 { 3011 unsigned long flags = s->flags; 3012 unsigned long size = s->object_size; 3013 int order; 3014 3015 /* 3016 * Round up object size to the next word boundary. We can only 3017 * place the free pointer at word boundaries and this determines 3018 * the possible location of the free pointer. 3019 */ 3020 size = ALIGN(size, sizeof(void *)); 3021 3022 #ifdef CONFIG_SLUB_DEBUG 3023 /* 3024 * Determine if we can poison the object itself. If the user of 3025 * the slab may touch the object after free or before allocation 3026 * then we should never poison the object itself. 3027 */ 3028 if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && 3029 !s->ctor) 3030 s->flags |= __OBJECT_POISON; 3031 else 3032 s->flags &= ~__OBJECT_POISON; 3033 3034 3035 /* 3036 * If we are Redzoning then check if there is some space between the 3037 * end of the object and the free pointer. If not then add an 3038 * additional word to have some bytes to store Redzone information. 3039 */ 3040 if ((flags & SLAB_RED_ZONE) && size == s->object_size) 3041 size += sizeof(void *); 3042 #endif 3043 3044 /* 3045 * With that we have determined the number of bytes in actual use 3046 * by the object. This is the potential offset to the free pointer. 3047 */ 3048 s->inuse = size; 3049 3050 if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || 3051 s->ctor)) { 3052 /* 3053 * Relocate free pointer after the object if it is not 3054 * permitted to overwrite the first word of the object on 3055 * kmem_cache_free. 3056 * 3057 * This is the case if we do RCU, have a constructor or 3058 * destructor or are poisoning the objects. 3059 */ 3060 s->offset = size; 3061 size += sizeof(void *); 3062 } 3063 3064 #ifdef CONFIG_SLUB_DEBUG 3065 if (flags & SLAB_STORE_USER) 3066 /* 3067 * Need to store information about allocs and frees after 3068 * the object. 3069 */ 3070 size += 2 * sizeof(struct track); 3071 3072 if (flags & SLAB_RED_ZONE) 3073 /* 3074 * Add some empty padding so that we can catch 3075 * overwrites from earlier objects rather than let 3076 * tracking information or the free pointer be 3077 * corrupted if a user writes before the start 3078 * of the object. 3079 */ 3080 size += sizeof(void *); 3081 #endif 3082 3083 /* 3084 * SLUB stores one object immediately after another beginning from 3085 * offset 0. In order to align the objects we have to simply size 3086 * each object to conform to the alignment. 3087 */ 3088 size = ALIGN(size, s->align); 3089 s->size = size; 3090 if (forced_order >= 0) 3091 order = forced_order; 3092 else 3093 order = calculate_order(size, s->reserved); 3094 3095 if (order < 0) 3096 return 0; 3097 3098 s->allocflags = 0; 3099 if (order) 3100 s->allocflags |= __GFP_COMP; 3101 3102 if (s->flags & SLAB_CACHE_DMA) 3103 s->allocflags |= GFP_DMA; 3104 3105 if (s->flags & SLAB_RECLAIM_ACCOUNT) 3106 s->allocflags |= __GFP_RECLAIMABLE; 3107 3108 /* 3109 * Determine the number of objects per slab 3110 */ 3111 s->oo = oo_make(order, size, s->reserved); 3112 s->min = oo_make(get_order(size), size, s->reserved); 3113 if (oo_objects(s->oo) > oo_objects(s->max)) 3114 s->max = s->oo; 3115 3116 return !!oo_objects(s->oo); 3117 } 3118 3119 static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) 3120 { 3121 s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); 3122 s->reserved = 0; 3123 3124 if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU)) 3125 s->reserved = sizeof(struct rcu_head); 3126 3127 if (!calculate_sizes(s, -1)) 3128 goto error; 3129 if (disable_higher_order_debug) { 3130 /* 3131 * Disable debugging flags that store metadata if the min slab 3132 * order increased. 3133 */ 3134 if (get_order(s->size) > get_order(s->object_size)) { 3135 s->flags &= ~DEBUG_METADATA_FLAGS; 3136 s->offset = 0; 3137 if (!calculate_sizes(s, -1)) 3138 goto error; 3139 } 3140 } 3141 3142 #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ 3143 defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) 3144 if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) 3145 /* Enable fast mode */ 3146 s->flags |= __CMPXCHG_DOUBLE; 3147 #endif 3148 3149 /* 3150 * The larger the object size is, the more pages we want on the partial 3151 * list to avoid pounding the page allocator excessively. 3152 */ 3153 set_min_partial(s, ilog2(s->size) / 2); 3154 3155 /* 3156 * cpu_partial determined the maximum number of objects kept in the 3157 * per cpu partial lists of a processor. 3158 * 3159 * Per cpu partial lists mainly contain slabs that just have one 3160 * object freed. If they are used for allocation then they can be 3161 * filled up again with minimal effort. The slab will never hit the 3162 * per node partial lists and therefore no locking will be required. 3163 * 3164 * This setting also determines 3165 * 3166 * A) The number of objects from per cpu partial slabs dumped to the 3167 * per node list when we reach the limit. 3168 * B) The number of objects in cpu partial slabs to extract from the 3169 * per node list when we run out of per cpu objects. We only fetch 3170 * 50% to keep some capacity around for frees. 3171 */ 3172 if (!kmem_cache_has_cpu_partial(s)) 3173 s->cpu_partial = 0; 3174 else if (s->size >= PAGE_SIZE) 3175 s->cpu_partial = 2; 3176 else if (s->size >= 1024) 3177 s->cpu_partial = 6; 3178 else if (s->size >= 256) 3179 s->cpu_partial = 13; 3180 else 3181 s->cpu_partial = 30; 3182 3183 #ifdef CONFIG_NUMA 3184 s->remote_node_defrag_ratio = 1000; 3185 #endif 3186 if (!init_kmem_cache_nodes(s)) 3187 goto error; 3188 3189 if (alloc_kmem_cache_cpus(s)) 3190 return 0; 3191 3192 free_kmem_cache_nodes(s); 3193 error: 3194 if (flags & SLAB_PANIC) 3195 panic("Cannot create slab %s size=%lu realsize=%u " 3196 "order=%u offset=%u flags=%lx\n", 3197 s->name, (unsigned long)s->size, s->size, 3198 oo_order(s->oo), s->offset, flags); 3199 return -EINVAL; 3200 } 3201 3202 static void list_slab_objects(struct kmem_cache *s, struct page *page, 3203 const char *text) 3204 { 3205 #ifdef CONFIG_SLUB_DEBUG 3206 void *addr = page_address(page); 3207 void *p; 3208 unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) * 3209 sizeof(long), GFP_ATOMIC); 3210 if (!map) 3211 return; 3212 slab_err(s, page, text, s->name); 3213 slab_lock(page); 3214 3215 get_map(s, page, map); 3216 for_each_object(p, s, addr, page->objects) { 3217 3218 if (!test_bit(slab_index(p, s, addr), map)) { 3219 pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); 3220 print_tracking(s, p); 3221 } 3222 } 3223 slab_unlock(page); 3224 kfree(map); 3225 #endif 3226 } 3227 3228 /* 3229 * Attempt to free all partial slabs on a node. 3230 * This is called from kmem_cache_close(). We must be the last thread 3231 * using the cache and therefore we do not need to lock anymore. 3232 */ 3233 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) 3234 { 3235 struct page *page, *h; 3236 3237 list_for_each_entry_safe(page, h, &n->partial, lru) { 3238 if (!page->inuse) { 3239 __remove_partial(n, page); 3240 discard_slab(s, page); 3241 } else { 3242 list_slab_objects(s, page, 3243 "Objects remaining in %s on kmem_cache_close()"); 3244 } 3245 } 3246 } 3247 3248 /* 3249 * Release all resources used by a slab cache. 3250 */ 3251 static inline int kmem_cache_close(struct kmem_cache *s) 3252 { 3253 int node; 3254 struct kmem_cache_node *n; 3255 3256 flush_all(s); 3257 /* Attempt to free all objects */ 3258 for_each_kmem_cache_node(s, node, n) { 3259 free_partial(s, n); 3260 if (n->nr_partial || slabs_node(s, node)) 3261 return 1; 3262 } 3263 free_percpu(s->cpu_slab); 3264 free_kmem_cache_nodes(s); 3265 return 0; 3266 } 3267 3268 int __kmem_cache_shutdown(struct kmem_cache *s) 3269 { 3270 return kmem_cache_close(s); 3271 } 3272 3273 /******************************************************************** 3274 * Kmalloc subsystem 3275 *******************************************************************/ 3276 3277 static int __init setup_slub_min_order(char *str) 3278 { 3279 get_option(&str, &slub_min_order); 3280 3281 return 1; 3282 } 3283 3284 __setup("slub_min_order=", setup_slub_min_order); 3285 3286 static int __init setup_slub_max_order(char *str) 3287 { 3288 get_option(&str, &slub_max_order); 3289 slub_max_order = min(slub_max_order, MAX_ORDER - 1); 3290 3291 return 1; 3292 } 3293 3294 __setup("slub_max_order=", setup_slub_max_order); 3295 3296 static int __init setup_slub_min_objects(char *str) 3297 { 3298 get_option(&str, &slub_min_objects); 3299 3300 return 1; 3301 } 3302 3303 __setup("slub_min_objects=", setup_slub_min_objects); 3304 3305 void *__kmalloc(size_t size, gfp_t flags) 3306 { 3307 struct kmem_cache *s; 3308 void *ret; 3309 3310 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 3311 return kmalloc_large(size, flags); 3312 3313 s = kmalloc_slab(size, flags); 3314 3315 if (unlikely(ZERO_OR_NULL_PTR(s))) 3316 return s; 3317 3318 ret = slab_alloc(s, flags, _RET_IP_); 3319 3320 trace_kmalloc(_RET_IP_, ret, size, s->size, flags); 3321 3322 kasan_kmalloc(s, ret, size); 3323 3324 return ret; 3325 } 3326 EXPORT_SYMBOL(__kmalloc); 3327 3328 #ifdef CONFIG_NUMA 3329 static void *kmalloc_large_node(size_t size, gfp_t flags, int node) 3330 { 3331 struct page *page; 3332 void *ptr = NULL; 3333 3334 flags |= __GFP_COMP | __GFP_NOTRACK; 3335 page = alloc_kmem_pages_node(node, flags, get_order(size)); 3336 if (page) 3337 ptr = page_address(page); 3338 3339 kmalloc_large_node_hook(ptr, size, flags); 3340 return ptr; 3341 } 3342 3343 void *__kmalloc_node(size_t size, gfp_t flags, int node) 3344 { 3345 struct kmem_cache *s; 3346 void *ret; 3347 3348 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 3349 ret = kmalloc_large_node(size, flags, node); 3350 3351 trace_kmalloc_node(_RET_IP_, ret, 3352 size, PAGE_SIZE << get_order(size), 3353 flags, node); 3354 3355 return ret; 3356 } 3357 3358 s = kmalloc_slab(size, flags); 3359 3360 if (unlikely(ZERO_OR_NULL_PTR(s))) 3361 return s; 3362 3363 ret = slab_alloc_node(s, flags, node, _RET_IP_); 3364 3365 trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); 3366 3367 kasan_kmalloc(s, ret, size); 3368 3369 return ret; 3370 } 3371 EXPORT_SYMBOL(__kmalloc_node); 3372 #endif 3373 3374 static size_t __ksize(const void *object) 3375 { 3376 struct page *page; 3377 3378 if (unlikely(object == ZERO_SIZE_PTR)) 3379 return 0; 3380 3381 page = virt_to_head_page(object); 3382 3383 if (unlikely(!PageSlab(page))) { 3384 WARN_ON(!PageCompound(page)); 3385 return PAGE_SIZE << compound_order(page); 3386 } 3387 3388 return slab_ksize(page->slab_cache); 3389 } 3390 3391 size_t ksize(const void *object) 3392 { 3393 size_t size = __ksize(object); 3394 /* We assume that ksize callers could use whole allocated area, 3395 so we need unpoison this area. */ 3396 kasan_krealloc(object, size); 3397 return size; 3398 } 3399 EXPORT_SYMBOL(ksize); 3400 3401 void kfree(const void *x) 3402 { 3403 struct page *page; 3404 void *object = (void *)x; 3405 3406 trace_kfree(_RET_IP_, x); 3407 3408 if (unlikely(ZERO_OR_NULL_PTR(x))) 3409 return; 3410 3411 page = virt_to_head_page(x); 3412 if (unlikely(!PageSlab(page))) { 3413 BUG_ON(!PageCompound(page)); 3414 kfree_hook(x); 3415 __free_kmem_pages(page, compound_order(page)); 3416 return; 3417 } 3418 slab_free(page->slab_cache, page, object, _RET_IP_); 3419 } 3420 EXPORT_SYMBOL(kfree); 3421 3422 #define SHRINK_PROMOTE_MAX 32 3423 3424 /* 3425 * kmem_cache_shrink discards empty slabs and promotes the slabs filled 3426 * up most to the head of the partial lists. New allocations will then 3427 * fill those up and thus they can be removed from the partial lists. 3428 * 3429 * The slabs with the least items are placed last. This results in them 3430 * being allocated from last increasing the chance that the last objects 3431 * are freed in them. 3432 */ 3433 int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate) 3434 { 3435 int node; 3436 int i; 3437 struct kmem_cache_node *n; 3438 struct page *page; 3439 struct page *t; 3440 struct list_head discard; 3441 struct list_head promote[SHRINK_PROMOTE_MAX]; 3442 unsigned long flags; 3443 int ret = 0; 3444 3445 if (deactivate) { 3446 /* 3447 * Disable empty slabs caching. Used to avoid pinning offline 3448 * memory cgroups by kmem pages that can be freed. 3449 */ 3450 s->cpu_partial = 0; 3451 s->min_partial = 0; 3452 3453 /* 3454 * s->cpu_partial is checked locklessly (see put_cpu_partial), 3455 * so we have to make sure the change is visible. 3456 */ 3457 kick_all_cpus_sync(); 3458 } 3459 3460 flush_all(s); 3461 for_each_kmem_cache_node(s, node, n) { 3462 INIT_LIST_HEAD(&discard); 3463 for (i = 0; i < SHRINK_PROMOTE_MAX; i++) 3464 INIT_LIST_HEAD(promote + i); 3465 3466 spin_lock_irqsave(&n->list_lock, flags); 3467 3468 /* 3469 * Build lists of slabs to discard or promote. 3470 * 3471 * Note that concurrent frees may occur while we hold the 3472 * list_lock. page->inuse here is the upper limit. 3473 */ 3474 list_for_each_entry_safe(page, t, &n->partial, lru) { 3475 int free = page->objects - page->inuse; 3476 3477 /* Do not reread page->inuse */ 3478 barrier(); 3479 3480 /* We do not keep full slabs on the list */ 3481 BUG_ON(free <= 0); 3482 3483 if (free == page->objects) { 3484 list_move(&page->lru, &discard); 3485 n->nr_partial--; 3486 } else if (free <= SHRINK_PROMOTE_MAX) 3487 list_move(&page->lru, promote + free - 1); 3488 } 3489 3490 /* 3491 * Promote the slabs filled up most to the head of the 3492 * partial list. 3493 */ 3494 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) 3495 list_splice(promote + i, &n->partial); 3496 3497 spin_unlock_irqrestore(&n->list_lock, flags); 3498 3499 /* Release empty slabs */ 3500 list_for_each_entry_safe(page, t, &discard, lru) 3501 discard_slab(s, page); 3502 3503 if (slabs_node(s, node)) 3504 ret = 1; 3505 } 3506 3507 return ret; 3508 } 3509 3510 static int slab_mem_going_offline_callback(void *arg) 3511 { 3512 struct kmem_cache *s; 3513 3514 mutex_lock(&slab_mutex); 3515 list_for_each_entry(s, &slab_caches, list) 3516 __kmem_cache_shrink(s, false); 3517 mutex_unlock(&slab_mutex); 3518 3519 return 0; 3520 } 3521 3522 static void slab_mem_offline_callback(void *arg) 3523 { 3524 struct kmem_cache_node *n; 3525 struct kmem_cache *s; 3526 struct memory_notify *marg = arg; 3527 int offline_node; 3528 3529 offline_node = marg->status_change_nid_normal; 3530 3531 /* 3532 * If the node still has available memory. we need kmem_cache_node 3533 * for it yet. 3534 */ 3535 if (offline_node < 0) 3536 return; 3537 3538 mutex_lock(&slab_mutex); 3539 list_for_each_entry(s, &slab_caches, list) { 3540 n = get_node(s, offline_node); 3541 if (n) { 3542 /* 3543 * if n->nr_slabs > 0, slabs still exist on the node 3544 * that is going down. We were unable to free them, 3545 * and offline_pages() function shouldn't call this 3546 * callback. So, we must fail. 3547 */ 3548 BUG_ON(slabs_node(s, offline_node)); 3549 3550 s->node[offline_node] = NULL; 3551 kmem_cache_free(kmem_cache_node, n); 3552 } 3553 } 3554 mutex_unlock(&slab_mutex); 3555 } 3556 3557 static int slab_mem_going_online_callback(void *arg) 3558 { 3559 struct kmem_cache_node *n; 3560 struct kmem_cache *s; 3561 struct memory_notify *marg = arg; 3562 int nid = marg->status_change_nid_normal; 3563 int ret = 0; 3564 3565 /* 3566 * If the node's memory is already available, then kmem_cache_node is 3567 * already created. Nothing to do. 3568 */ 3569 if (nid < 0) 3570 return 0; 3571 3572 /* 3573 * We are bringing a node online. No memory is available yet. We must 3574 * allocate a kmem_cache_node structure in order to bring the node 3575 * online. 3576 */ 3577 mutex_lock(&slab_mutex); 3578 list_for_each_entry(s, &slab_caches, list) { 3579 /* 3580 * XXX: kmem_cache_alloc_node will fallback to other nodes 3581 * since memory is not yet available from the node that 3582 * is brought up. 3583 */ 3584 n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); 3585 if (!n) { 3586 ret = -ENOMEM; 3587 goto out; 3588 } 3589 init_kmem_cache_node(n); 3590 s->node[nid] = n; 3591 } 3592 out: 3593 mutex_unlock(&slab_mutex); 3594 return ret; 3595 } 3596 3597 static int slab_memory_callback(struct notifier_block *self, 3598 unsigned long action, void *arg) 3599 { 3600 int ret = 0; 3601 3602 switch (action) { 3603 case MEM_GOING_ONLINE: 3604 ret = slab_mem_going_online_callback(arg); 3605 break; 3606 case MEM_GOING_OFFLINE: 3607 ret = slab_mem_going_offline_callback(arg); 3608 break; 3609 case MEM_OFFLINE: 3610 case MEM_CANCEL_ONLINE: 3611 slab_mem_offline_callback(arg); 3612 break; 3613 case MEM_ONLINE: 3614 case MEM_CANCEL_OFFLINE: 3615 break; 3616 } 3617 if (ret) 3618 ret = notifier_from_errno(ret); 3619 else 3620 ret = NOTIFY_OK; 3621 return ret; 3622 } 3623 3624 static struct notifier_block slab_memory_callback_nb = { 3625 .notifier_call = slab_memory_callback, 3626 .priority = SLAB_CALLBACK_PRI, 3627 }; 3628 3629 /******************************************************************** 3630 * Basic setup of slabs 3631 *******************************************************************/ 3632 3633 /* 3634 * Used for early kmem_cache structures that were allocated using 3635 * the page allocator. Allocate them properly then fix up the pointers 3636 * that may be pointing to the wrong kmem_cache structure. 3637 */ 3638 3639 static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) 3640 { 3641 int node; 3642 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 3643 struct kmem_cache_node *n; 3644 3645 memcpy(s, static_cache, kmem_cache->object_size); 3646 3647 /* 3648 * This runs very early, and only the boot processor is supposed to be 3649 * up. Even if it weren't true, IRQs are not up so we couldn't fire 3650 * IPIs around. 3651 */ 3652 __flush_cpu_slab(s, smp_processor_id()); 3653 for_each_kmem_cache_node(s, node, n) { 3654 struct page *p; 3655 3656 list_for_each_entry(p, &n->partial, lru) 3657 p->slab_cache = s; 3658 3659 #ifdef CONFIG_SLUB_DEBUG 3660 list_for_each_entry(p, &n->full, lru) 3661 p->slab_cache = s; 3662 #endif 3663 } 3664 slab_init_memcg_params(s); 3665 list_add(&s->list, &slab_caches); 3666 return s; 3667 } 3668 3669 void __init kmem_cache_init(void) 3670 { 3671 static __initdata struct kmem_cache boot_kmem_cache, 3672 boot_kmem_cache_node; 3673 3674 if (debug_guardpage_minorder()) 3675 slub_max_order = 0; 3676 3677 kmem_cache_node = &boot_kmem_cache_node; 3678 kmem_cache = &boot_kmem_cache; 3679 3680 create_boot_cache(kmem_cache_node, "kmem_cache_node", 3681 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN); 3682 3683 register_hotmemory_notifier(&slab_memory_callback_nb); 3684 3685 /* Able to allocate the per node structures */ 3686 slab_state = PARTIAL; 3687 3688 create_boot_cache(kmem_cache, "kmem_cache", 3689 offsetof(struct kmem_cache, node) + 3690 nr_node_ids * sizeof(struct kmem_cache_node *), 3691 SLAB_HWCACHE_ALIGN); 3692 3693 kmem_cache = bootstrap(&boot_kmem_cache); 3694 3695 /* 3696 * Allocate kmem_cache_node properly from the kmem_cache slab. 3697 * kmem_cache_node is separately allocated so no need to 3698 * update any list pointers. 3699 */ 3700 kmem_cache_node = bootstrap(&boot_kmem_cache_node); 3701 3702 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 3703 create_kmalloc_caches(0); 3704 3705 #ifdef CONFIG_SMP 3706 register_cpu_notifier(&slab_notifier); 3707 #endif 3708 3709 pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n", 3710 cache_line_size(), 3711 slub_min_order, slub_max_order, slub_min_objects, 3712 nr_cpu_ids, nr_node_ids); 3713 } 3714 3715 void __init kmem_cache_init_late(void) 3716 { 3717 } 3718 3719 struct kmem_cache * 3720 __kmem_cache_alias(const char *name, size_t size, size_t align, 3721 unsigned long flags, void (*ctor)(void *)) 3722 { 3723 struct kmem_cache *s, *c; 3724 3725 s = find_mergeable(size, align, flags, name, ctor); 3726 if (s) { 3727 s->refcount++; 3728 3729 /* 3730 * Adjust the object sizes so that we clear 3731 * the complete object on kzalloc. 3732 */ 3733 s->object_size = max(s->object_size, (int)size); 3734 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); 3735 3736 for_each_memcg_cache(c, s) { 3737 c->object_size = s->object_size; 3738 c->inuse = max_t(int, c->inuse, 3739 ALIGN(size, sizeof(void *))); 3740 } 3741 3742 if (sysfs_slab_alias(s, name)) { 3743 s->refcount--; 3744 s = NULL; 3745 } 3746 } 3747 3748 return s; 3749 } 3750 3751 int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) 3752 { 3753 int err; 3754 3755 err = kmem_cache_open(s, flags); 3756 if (err) 3757 return err; 3758 3759 /* Mutex is not taken during early boot */ 3760 if (slab_state <= UP) 3761 return 0; 3762 3763 memcg_propagate_slab_attrs(s); 3764 err = sysfs_slab_add(s); 3765 if (err) 3766 kmem_cache_close(s); 3767 3768 return err; 3769 } 3770 3771 #ifdef CONFIG_SMP 3772 /* 3773 * Use the cpu notifier to insure that the cpu slabs are flushed when 3774 * necessary. 3775 */ 3776 static int slab_cpuup_callback(struct notifier_block *nfb, 3777 unsigned long action, void *hcpu) 3778 { 3779 long cpu = (long)hcpu; 3780 struct kmem_cache *s; 3781 unsigned long flags; 3782 3783 switch (action) { 3784 case CPU_UP_CANCELED: 3785 case CPU_UP_CANCELED_FROZEN: 3786 case CPU_DEAD: 3787 case CPU_DEAD_FROZEN: 3788 mutex_lock(&slab_mutex); 3789 list_for_each_entry(s, &slab_caches, list) { 3790 local_irq_save(flags); 3791 __flush_cpu_slab(s, cpu); 3792 local_irq_restore(flags); 3793 } 3794 mutex_unlock(&slab_mutex); 3795 break; 3796 default: 3797 break; 3798 } 3799 return NOTIFY_OK; 3800 } 3801 3802 static struct notifier_block slab_notifier = { 3803 .notifier_call = slab_cpuup_callback 3804 }; 3805 3806 #endif 3807 3808 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) 3809 { 3810 struct kmem_cache *s; 3811 void *ret; 3812 3813 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) 3814 return kmalloc_large(size, gfpflags); 3815 3816 s = kmalloc_slab(size, gfpflags); 3817 3818 if (unlikely(ZERO_OR_NULL_PTR(s))) 3819 return s; 3820 3821 ret = slab_alloc(s, gfpflags, caller); 3822 3823 /* Honor the call site pointer we received. */ 3824 trace_kmalloc(caller, ret, size, s->size, gfpflags); 3825 3826 return ret; 3827 } 3828 3829 #ifdef CONFIG_NUMA 3830 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, 3831 int node, unsigned long caller) 3832 { 3833 struct kmem_cache *s; 3834 void *ret; 3835 3836 if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { 3837 ret = kmalloc_large_node(size, gfpflags, node); 3838 3839 trace_kmalloc_node(caller, ret, 3840 size, PAGE_SIZE << get_order(size), 3841 gfpflags, node); 3842 3843 return ret; 3844 } 3845 3846 s = kmalloc_slab(size, gfpflags); 3847 3848 if (unlikely(ZERO_OR_NULL_PTR(s))) 3849 return s; 3850 3851 ret = slab_alloc_node(s, gfpflags, node, caller); 3852 3853 /* Honor the call site pointer we received. */ 3854 trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); 3855 3856 return ret; 3857 } 3858 #endif 3859 3860 #ifdef CONFIG_SYSFS 3861 static int count_inuse(struct page *page) 3862 { 3863 return page->inuse; 3864 } 3865 3866 static int count_total(struct page *page) 3867 { 3868 return page->objects; 3869 } 3870 #endif 3871 3872 #ifdef CONFIG_SLUB_DEBUG 3873 static int validate_slab(struct kmem_cache *s, struct page *page, 3874 unsigned long *map) 3875 { 3876 void *p; 3877 void *addr = page_address(page); 3878 3879 if (!check_slab(s, page) || 3880 !on_freelist(s, page, NULL)) 3881 return 0; 3882 3883 /* Now we know that a valid freelist exists */ 3884 bitmap_zero(map, page->objects); 3885 3886 get_map(s, page, map); 3887 for_each_object(p, s, addr, page->objects) { 3888 if (test_bit(slab_index(p, s, addr), map)) 3889 if (!check_object(s, page, p, SLUB_RED_INACTIVE)) 3890 return 0; 3891 } 3892 3893 for_each_object(p, s, addr, page->objects) 3894 if (!test_bit(slab_index(p, s, addr), map)) 3895 if (!check_object(s, page, p, SLUB_RED_ACTIVE)) 3896 return 0; 3897 return 1; 3898 } 3899 3900 static void validate_slab_slab(struct kmem_cache *s, struct page *page, 3901 unsigned long *map) 3902 { 3903 slab_lock(page); 3904 validate_slab(s, page, map); 3905 slab_unlock(page); 3906 } 3907 3908 static int validate_slab_node(struct kmem_cache *s, 3909 struct kmem_cache_node *n, unsigned long *map) 3910 { 3911 unsigned long count = 0; 3912 struct page *page; 3913 unsigned long flags; 3914 3915 spin_lock_irqsave(&n->list_lock, flags); 3916 3917 list_for_each_entry(page, &n->partial, lru) { 3918 validate_slab_slab(s, page, map); 3919 count++; 3920 } 3921 if (count != n->nr_partial) 3922 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", 3923 s->name, count, n->nr_partial); 3924 3925 if (!(s->flags & SLAB_STORE_USER)) 3926 goto out; 3927 3928 list_for_each_entry(page, &n->full, lru) { 3929 validate_slab_slab(s, page, map); 3930 count++; 3931 } 3932 if (count != atomic_long_read(&n->nr_slabs)) 3933 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", 3934 s->name, count, atomic_long_read(&n->nr_slabs)); 3935 3936 out: 3937 spin_unlock_irqrestore(&n->list_lock, flags); 3938 return count; 3939 } 3940 3941 static long validate_slab_cache(struct kmem_cache *s) 3942 { 3943 int node; 3944 unsigned long count = 0; 3945 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * 3946 sizeof(unsigned long), GFP_KERNEL); 3947 struct kmem_cache_node *n; 3948 3949 if (!map) 3950 return -ENOMEM; 3951 3952 flush_all(s); 3953 for_each_kmem_cache_node(s, node, n) 3954 count += validate_slab_node(s, n, map); 3955 kfree(map); 3956 return count; 3957 } 3958 /* 3959 * Generate lists of code addresses where slabcache objects are allocated 3960 * and freed. 3961 */ 3962 3963 struct location { 3964 unsigned long count; 3965 unsigned long addr; 3966 long long sum_time; 3967 long min_time; 3968 long max_time; 3969 long min_pid; 3970 long max_pid; 3971 DECLARE_BITMAP(cpus, NR_CPUS); 3972 nodemask_t nodes; 3973 }; 3974 3975 struct loc_track { 3976 unsigned long max; 3977 unsigned long count; 3978 struct location *loc; 3979 }; 3980 3981 static void free_loc_track(struct loc_track *t) 3982 { 3983 if (t->max) 3984 free_pages((unsigned long)t->loc, 3985 get_order(sizeof(struct location) * t->max)); 3986 } 3987 3988 static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) 3989 { 3990 struct location *l; 3991 int order; 3992 3993 order = get_order(sizeof(struct location) * max); 3994 3995 l = (void *)__get_free_pages(flags, order); 3996 if (!l) 3997 return 0; 3998 3999 if (t->count) { 4000 memcpy(l, t->loc, sizeof(struct location) * t->count); 4001 free_loc_track(t); 4002 } 4003 t->max = max; 4004 t->loc = l; 4005 return 1; 4006 } 4007 4008 static int add_location(struct loc_track *t, struct kmem_cache *s, 4009 const struct track *track) 4010 { 4011 long start, end, pos; 4012 struct location *l; 4013 unsigned long caddr; 4014 unsigned long age = jiffies - track->when; 4015 4016 start = -1; 4017 end = t->count; 4018 4019 for ( ; ; ) { 4020 pos = start + (end - start + 1) / 2; 4021 4022 /* 4023 * There is nothing at "end". If we end up there 4024 * we need to add something to before end. 4025 */ 4026 if (pos == end) 4027 break; 4028 4029 caddr = t->loc[pos].addr; 4030 if (track->addr == caddr) { 4031 4032 l = &t->loc[pos]; 4033 l->count++; 4034 if (track->when) { 4035 l->sum_time += age; 4036 if (age < l->min_time) 4037 l->min_time = age; 4038 if (age > l->max_time) 4039 l->max_time = age; 4040 4041 if (track->pid < l->min_pid) 4042 l->min_pid = track->pid; 4043 if (track->pid > l->max_pid) 4044 l->max_pid = track->pid; 4045 4046 cpumask_set_cpu(track->cpu, 4047 to_cpumask(l->cpus)); 4048 } 4049 node_set(page_to_nid(virt_to_page(track)), l->nodes); 4050 return 1; 4051 } 4052 4053 if (track->addr < caddr) 4054 end = pos; 4055 else 4056 start = pos; 4057 } 4058 4059 /* 4060 * Not found. Insert new tracking element. 4061 */ 4062 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) 4063 return 0; 4064 4065 l = t->loc + pos; 4066 if (pos < t->count) 4067 memmove(l + 1, l, 4068 (t->count - pos) * sizeof(struct location)); 4069 t->count++; 4070 l->count = 1; 4071 l->addr = track->addr; 4072 l->sum_time = age; 4073 l->min_time = age; 4074 l->max_time = age; 4075 l->min_pid = track->pid; 4076 l->max_pid = track->pid; 4077 cpumask_clear(to_cpumask(l->cpus)); 4078 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); 4079 nodes_clear(l->nodes); 4080 node_set(page_to_nid(virt_to_page(track)), l->nodes); 4081 return 1; 4082 } 4083 4084 static void process_slab(struct loc_track *t, struct kmem_cache *s, 4085 struct page *page, enum track_item alloc, 4086 unsigned long *map) 4087 { 4088 void *addr = page_address(page); 4089 void *p; 4090 4091 bitmap_zero(map, page->objects); 4092 get_map(s, page, map); 4093 4094 for_each_object(p, s, addr, page->objects) 4095 if (!test_bit(slab_index(p, s, addr), map)) 4096 add_location(t, s, get_track(s, p, alloc)); 4097 } 4098 4099 static int list_locations(struct kmem_cache *s, char *buf, 4100 enum track_item alloc) 4101 { 4102 int len = 0; 4103 unsigned long i; 4104 struct loc_track t = { 0, 0, NULL }; 4105 int node; 4106 unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * 4107 sizeof(unsigned long), GFP_KERNEL); 4108 struct kmem_cache_node *n; 4109 4110 if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), 4111 GFP_TEMPORARY)) { 4112 kfree(map); 4113 return sprintf(buf, "Out of memory\n"); 4114 } 4115 /* Push back cpu slabs */ 4116 flush_all(s); 4117 4118 for_each_kmem_cache_node(s, node, n) { 4119 unsigned long flags; 4120 struct page *page; 4121 4122 if (!atomic_long_read(&n->nr_slabs)) 4123 continue; 4124 4125 spin_lock_irqsave(&n->list_lock, flags); 4126 list_for_each_entry(page, &n->partial, lru) 4127 process_slab(&t, s, page, alloc, map); 4128 list_for_each_entry(page, &n->full, lru) 4129 process_slab(&t, s, page, alloc, map); 4130 spin_unlock_irqrestore(&n->list_lock, flags); 4131 } 4132 4133 for (i = 0; i < t.count; i++) { 4134 struct location *l = &t.loc[i]; 4135 4136 if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) 4137 break; 4138 len += sprintf(buf + len, "%7ld ", l->count); 4139 4140 if (l->addr) 4141 len += sprintf(buf + len, "%pS", (void *)l->addr); 4142 else 4143 len += sprintf(buf + len, "<not-available>"); 4144 4145 if (l->sum_time != l->min_time) { 4146 len += sprintf(buf + len, " age=%ld/%ld/%ld", 4147 l->min_time, 4148 (long)div_u64(l->sum_time, l->count), 4149 l->max_time); 4150 } else 4151 len += sprintf(buf + len, " age=%ld", 4152 l->min_time); 4153 4154 if (l->min_pid != l->max_pid) 4155 len += sprintf(buf + len, " pid=%ld-%ld", 4156 l->min_pid, l->max_pid); 4157 else 4158 len += sprintf(buf + len, " pid=%ld", 4159 l->min_pid); 4160 4161 if (num_online_cpus() > 1 && 4162 !cpumask_empty(to_cpumask(l->cpus)) && 4163 len < PAGE_SIZE - 60) 4164 len += scnprintf(buf + len, PAGE_SIZE - len - 50, 4165 " cpus=%*pbl", 4166 cpumask_pr_args(to_cpumask(l->cpus))); 4167 4168 if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && 4169 len < PAGE_SIZE - 60) 4170 len += scnprintf(buf + len, PAGE_SIZE - len - 50, 4171 " nodes=%*pbl", 4172 nodemask_pr_args(&l->nodes)); 4173 4174 len += sprintf(buf + len, "\n"); 4175 } 4176 4177 free_loc_track(&t); 4178 kfree(map); 4179 if (!t.count) 4180 len += sprintf(buf, "No data\n"); 4181 return len; 4182 } 4183 #endif 4184 4185 #ifdef SLUB_RESILIENCY_TEST 4186 static void __init resiliency_test(void) 4187 { 4188 u8 *p; 4189 4190 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10); 4191 4192 pr_err("SLUB resiliency testing\n"); 4193 pr_err("-----------------------\n"); 4194 pr_err("A. Corruption after allocation\n"); 4195 4196 p = kzalloc(16, GFP_KERNEL); 4197 p[16] = 0x12; 4198 pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", 4199 p + 16); 4200 4201 validate_slab_cache(kmalloc_caches[4]); 4202 4203 /* Hmmm... The next two are dangerous */ 4204 p = kzalloc(32, GFP_KERNEL); 4205 p[32 + sizeof(void *)] = 0x34; 4206 pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", 4207 p); 4208 pr_err("If allocated object is overwritten then not detectable\n\n"); 4209 4210 validate_slab_cache(kmalloc_caches[5]); 4211 p = kzalloc(64, GFP_KERNEL); 4212 p += 64 + (get_cycles() & 0xff) * sizeof(void *); 4213 *p = 0x56; 4214 pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", 4215 p); 4216 pr_err("If allocated object is overwritten then not detectable\n\n"); 4217 validate_slab_cache(kmalloc_caches[6]); 4218 4219 pr_err("\nB. Corruption after free\n"); 4220 p = kzalloc(128, GFP_KERNEL); 4221 kfree(p); 4222 *p = 0x78; 4223 pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); 4224 validate_slab_cache(kmalloc_caches[7]); 4225 4226 p = kzalloc(256, GFP_KERNEL); 4227 kfree(p); 4228 p[50] = 0x9a; 4229 pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); 4230 validate_slab_cache(kmalloc_caches[8]); 4231 4232 p = kzalloc(512, GFP_KERNEL); 4233 kfree(p); 4234 p[512] = 0xab; 4235 pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); 4236 validate_slab_cache(kmalloc_caches[9]); 4237 } 4238 #else 4239 #ifdef CONFIG_SYSFS 4240 static void resiliency_test(void) {}; 4241 #endif 4242 #endif 4243 4244 #ifdef CONFIG_SYSFS 4245 enum slab_stat_type { 4246 SL_ALL, /* All slabs */ 4247 SL_PARTIAL, /* Only partially allocated slabs */ 4248 SL_CPU, /* Only slabs used for cpu caches */ 4249 SL_OBJECTS, /* Determine allocated objects not slabs */ 4250 SL_TOTAL /* Determine object capacity not slabs */ 4251 }; 4252 4253 #define SO_ALL (1 << SL_ALL) 4254 #define SO_PARTIAL (1 << SL_PARTIAL) 4255 #define SO_CPU (1 << SL_CPU) 4256 #define SO_OBJECTS (1 << SL_OBJECTS) 4257 #define SO_TOTAL (1 << SL_TOTAL) 4258 4259 static ssize_t show_slab_objects(struct kmem_cache *s, 4260 char *buf, unsigned long flags) 4261 { 4262 unsigned long total = 0; 4263 int node; 4264 int x; 4265 unsigned long *nodes; 4266 4267 nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); 4268 if (!nodes) 4269 return -ENOMEM; 4270 4271 if (flags & SO_CPU) { 4272 int cpu; 4273 4274 for_each_possible_cpu(cpu) { 4275 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, 4276 cpu); 4277 int node; 4278 struct page *page; 4279 4280 page = ACCESS_ONCE(c->page); 4281 if (!page) 4282 continue; 4283 4284 node = page_to_nid(page); 4285 if (flags & SO_TOTAL) 4286 x = page->objects; 4287 else if (flags & SO_OBJECTS) 4288 x = page->inuse; 4289 else 4290 x = 1; 4291 4292 total += x; 4293 nodes[node] += x; 4294 4295 page = ACCESS_ONCE(c->partial); 4296 if (page) { 4297 node = page_to_nid(page); 4298 if (flags & SO_TOTAL) 4299 WARN_ON_ONCE(1); 4300 else if (flags & SO_OBJECTS) 4301 WARN_ON_ONCE(1); 4302 else 4303 x = page->pages; 4304 total += x; 4305 nodes[node] += x; 4306 } 4307 } 4308 } 4309 4310 get_online_mems(); 4311 #ifdef CONFIG_SLUB_DEBUG 4312 if (flags & SO_ALL) { 4313 struct kmem_cache_node *n; 4314 4315 for_each_kmem_cache_node(s, node, n) { 4316 4317 if (flags & SO_TOTAL) 4318 x = atomic_long_read(&n->total_objects); 4319 else if (flags & SO_OBJECTS) 4320 x = atomic_long_read(&n->total_objects) - 4321 count_partial(n, count_free); 4322 else 4323 x = atomic_long_read(&n->nr_slabs); 4324 total += x; 4325 nodes[node] += x; 4326 } 4327 4328 } else 4329 #endif 4330 if (flags & SO_PARTIAL) { 4331 struct kmem_cache_node *n; 4332 4333 for_each_kmem_cache_node(s, node, n) { 4334 if (flags & SO_TOTAL) 4335 x = count_partial(n, count_total); 4336 else if (flags & SO_OBJECTS) 4337 x = count_partial(n, count_inuse); 4338 else 4339 x = n->nr_partial; 4340 total += x; 4341 nodes[node] += x; 4342 } 4343 } 4344 x = sprintf(buf, "%lu", total); 4345 #ifdef CONFIG_NUMA 4346 for (node = 0; node < nr_node_ids; node++) 4347 if (nodes[node]) 4348 x += sprintf(buf + x, " N%d=%lu", 4349 node, nodes[node]); 4350 #endif 4351 put_online_mems(); 4352 kfree(nodes); 4353 return x + sprintf(buf + x, "\n"); 4354 } 4355 4356 #ifdef CONFIG_SLUB_DEBUG 4357 static int any_slab_objects(struct kmem_cache *s) 4358 { 4359 int node; 4360 struct kmem_cache_node *n; 4361 4362 for_each_kmem_cache_node(s, node, n) 4363 if (atomic_long_read(&n->total_objects)) 4364 return 1; 4365 4366 return 0; 4367 } 4368 #endif 4369 4370 #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) 4371 #define to_slab(n) container_of(n, struct kmem_cache, kobj) 4372 4373 struct slab_attribute { 4374 struct attribute attr; 4375 ssize_t (*show)(struct kmem_cache *s, char *buf); 4376 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); 4377 }; 4378 4379 #define SLAB_ATTR_RO(_name) \ 4380 static struct slab_attribute _name##_attr = \ 4381 __ATTR(_name, 0400, _name##_show, NULL) 4382 4383 #define SLAB_ATTR(_name) \ 4384 static struct slab_attribute _name##_attr = \ 4385 __ATTR(_name, 0600, _name##_show, _name##_store) 4386 4387 static ssize_t slab_size_show(struct kmem_cache *s, char *buf) 4388 { 4389 return sprintf(buf, "%d\n", s->size); 4390 } 4391 SLAB_ATTR_RO(slab_size); 4392 4393 static ssize_t align_show(struct kmem_cache *s, char *buf) 4394 { 4395 return sprintf(buf, "%d\n", s->align); 4396 } 4397 SLAB_ATTR_RO(align); 4398 4399 static ssize_t object_size_show(struct kmem_cache *s, char *buf) 4400 { 4401 return sprintf(buf, "%d\n", s->object_size); 4402 } 4403 SLAB_ATTR_RO(object_size); 4404 4405 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) 4406 { 4407 return sprintf(buf, "%d\n", oo_objects(s->oo)); 4408 } 4409 SLAB_ATTR_RO(objs_per_slab); 4410 4411 static ssize_t order_store(struct kmem_cache *s, 4412 const char *buf, size_t length) 4413 { 4414 unsigned long order; 4415 int err; 4416 4417 err = kstrtoul(buf, 10, &order); 4418 if (err) 4419 return err; 4420 4421 if (order > slub_max_order || order < slub_min_order) 4422 return -EINVAL; 4423 4424 calculate_sizes(s, order); 4425 return length; 4426 } 4427 4428 static ssize_t order_show(struct kmem_cache *s, char *buf) 4429 { 4430 return sprintf(buf, "%d\n", oo_order(s->oo)); 4431 } 4432 SLAB_ATTR(order); 4433 4434 static ssize_t min_partial_show(struct kmem_cache *s, char *buf) 4435 { 4436 return sprintf(buf, "%lu\n", s->min_partial); 4437 } 4438 4439 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, 4440 size_t length) 4441 { 4442 unsigned long min; 4443 int err; 4444 4445 err = kstrtoul(buf, 10, &min); 4446 if (err) 4447 return err; 4448 4449 set_min_partial(s, min); 4450 return length; 4451 } 4452 SLAB_ATTR(min_partial); 4453 4454 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) 4455 { 4456 return sprintf(buf, "%u\n", s->cpu_partial); 4457 } 4458 4459 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, 4460 size_t length) 4461 { 4462 unsigned long objects; 4463 int err; 4464 4465 err = kstrtoul(buf, 10, &objects); 4466 if (err) 4467 return err; 4468 if (objects && !kmem_cache_has_cpu_partial(s)) 4469 return -EINVAL; 4470 4471 s->cpu_partial = objects; 4472 flush_all(s); 4473 return length; 4474 } 4475 SLAB_ATTR(cpu_partial); 4476 4477 static ssize_t ctor_show(struct kmem_cache *s, char *buf) 4478 { 4479 if (!s->ctor) 4480 return 0; 4481 return sprintf(buf, "%pS\n", s->ctor); 4482 } 4483 SLAB_ATTR_RO(ctor); 4484 4485 static ssize_t aliases_show(struct kmem_cache *s, char *buf) 4486 { 4487 return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); 4488 } 4489 SLAB_ATTR_RO(aliases); 4490 4491 static ssize_t partial_show(struct kmem_cache *s, char *buf) 4492 { 4493 return show_slab_objects(s, buf, SO_PARTIAL); 4494 } 4495 SLAB_ATTR_RO(partial); 4496 4497 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) 4498 { 4499 return show_slab_objects(s, buf, SO_CPU); 4500 } 4501 SLAB_ATTR_RO(cpu_slabs); 4502 4503 static ssize_t objects_show(struct kmem_cache *s, char *buf) 4504 { 4505 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); 4506 } 4507 SLAB_ATTR_RO(objects); 4508 4509 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) 4510 { 4511 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); 4512 } 4513 SLAB_ATTR_RO(objects_partial); 4514 4515 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) 4516 { 4517 int objects = 0; 4518 int pages = 0; 4519 int cpu; 4520 int len; 4521 4522 for_each_online_cpu(cpu) { 4523 struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; 4524 4525 if (page) { 4526 pages += page->pages; 4527 objects += page->pobjects; 4528 } 4529 } 4530 4531 len = sprintf(buf, "%d(%d)", objects, pages); 4532 4533 #ifdef CONFIG_SMP 4534 for_each_online_cpu(cpu) { 4535 struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; 4536 4537 if (page && len < PAGE_SIZE - 20) 4538 len += sprintf(buf + len, " C%d=%d(%d)", cpu, 4539 page->pobjects, page->pages); 4540 } 4541 #endif 4542 return len + sprintf(buf + len, "\n"); 4543 } 4544 SLAB_ATTR_RO(slabs_cpu_partial); 4545 4546 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) 4547 { 4548 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); 4549 } 4550 4551 static ssize_t reclaim_account_store(struct kmem_cache *s, 4552 const char *buf, size_t length) 4553 { 4554 s->flags &= ~SLAB_RECLAIM_ACCOUNT; 4555 if (buf[0] == '1') 4556 s->flags |= SLAB_RECLAIM_ACCOUNT; 4557 return length; 4558 } 4559 SLAB_ATTR(reclaim_account); 4560 4561 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) 4562 { 4563 return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); 4564 } 4565 SLAB_ATTR_RO(hwcache_align); 4566 4567 #ifdef CONFIG_ZONE_DMA 4568 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) 4569 { 4570 return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); 4571 } 4572 SLAB_ATTR_RO(cache_dma); 4573 #endif 4574 4575 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) 4576 { 4577 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); 4578 } 4579 SLAB_ATTR_RO(destroy_by_rcu); 4580 4581 static ssize_t reserved_show(struct kmem_cache *s, char *buf) 4582 { 4583 return sprintf(buf, "%d\n", s->reserved); 4584 } 4585 SLAB_ATTR_RO(reserved); 4586 4587 #ifdef CONFIG_SLUB_DEBUG 4588 static ssize_t slabs_show(struct kmem_cache *s, char *buf) 4589 { 4590 return show_slab_objects(s, buf, SO_ALL); 4591 } 4592 SLAB_ATTR_RO(slabs); 4593 4594 static ssize_t total_objects_show(struct kmem_cache *s, char *buf) 4595 { 4596 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); 4597 } 4598 SLAB_ATTR_RO(total_objects); 4599 4600 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) 4601 { 4602 return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); 4603 } 4604 4605 static ssize_t sanity_checks_store(struct kmem_cache *s, 4606 const char *buf, size_t length) 4607 { 4608 s->flags &= ~SLAB_DEBUG_FREE; 4609 if (buf[0] == '1') { 4610 s->flags &= ~__CMPXCHG_DOUBLE; 4611 s->flags |= SLAB_DEBUG_FREE; 4612 } 4613 return length; 4614 } 4615 SLAB_ATTR(sanity_checks); 4616 4617 static ssize_t trace_show(struct kmem_cache *s, char *buf) 4618 { 4619 return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); 4620 } 4621 4622 static ssize_t trace_store(struct kmem_cache *s, const char *buf, 4623 size_t length) 4624 { 4625 /* 4626 * Tracing a merged cache is going to give confusing results 4627 * as well as cause other issues like converting a mergeable 4628 * cache into an umergeable one. 4629 */ 4630 if (s->refcount > 1) 4631 return -EINVAL; 4632 4633 s->flags &= ~SLAB_TRACE; 4634 if (buf[0] == '1') { 4635 s->flags &= ~__CMPXCHG_DOUBLE; 4636 s->flags |= SLAB_TRACE; 4637 } 4638 return length; 4639 } 4640 SLAB_ATTR(trace); 4641 4642 static ssize_t red_zone_show(struct kmem_cache *s, char *buf) 4643 { 4644 return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); 4645 } 4646 4647 static ssize_t red_zone_store(struct kmem_cache *s, 4648 const char *buf, size_t length) 4649 { 4650 if (any_slab_objects(s)) 4651 return -EBUSY; 4652 4653 s->flags &= ~SLAB_RED_ZONE; 4654 if (buf[0] == '1') { 4655 s->flags &= ~__CMPXCHG_DOUBLE; 4656 s->flags |= SLAB_RED_ZONE; 4657 } 4658 calculate_sizes(s, -1); 4659 return length; 4660 } 4661 SLAB_ATTR(red_zone); 4662 4663 static ssize_t poison_show(struct kmem_cache *s, char *buf) 4664 { 4665 return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); 4666 } 4667 4668 static ssize_t poison_store(struct kmem_cache *s, 4669 const char *buf, size_t length) 4670 { 4671 if (any_slab_objects(s)) 4672 return -EBUSY; 4673 4674 s->flags &= ~SLAB_POISON; 4675 if (buf[0] == '1') { 4676 s->flags &= ~__CMPXCHG_DOUBLE; 4677 s->flags |= SLAB_POISON; 4678 } 4679 calculate_sizes(s, -1); 4680 return length; 4681 } 4682 SLAB_ATTR(poison); 4683 4684 static ssize_t store_user_show(struct kmem_cache *s, char *buf) 4685 { 4686 return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); 4687 } 4688 4689 static ssize_t store_user_store(struct kmem_cache *s, 4690 const char *buf, size_t length) 4691 { 4692 if (any_slab_objects(s)) 4693 return -EBUSY; 4694 4695 s->flags &= ~SLAB_STORE_USER; 4696 if (buf[0] == '1') { 4697 s->flags &= ~__CMPXCHG_DOUBLE; 4698 s->flags |= SLAB_STORE_USER; 4699 } 4700 calculate_sizes(s, -1); 4701 return length; 4702 } 4703 SLAB_ATTR(store_user); 4704 4705 static ssize_t validate_show(struct kmem_cache *s, char *buf) 4706 { 4707 return 0; 4708 } 4709 4710 static ssize_t validate_store(struct kmem_cache *s, 4711 const char *buf, size_t length) 4712 { 4713 int ret = -EINVAL; 4714 4715 if (buf[0] == '1') { 4716 ret = validate_slab_cache(s); 4717 if (ret >= 0) 4718 ret = length; 4719 } 4720 return ret; 4721 } 4722 SLAB_ATTR(validate); 4723 4724 static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) 4725 { 4726 if (!(s->flags & SLAB_STORE_USER)) 4727 return -ENOSYS; 4728 return list_locations(s, buf, TRACK_ALLOC); 4729 } 4730 SLAB_ATTR_RO(alloc_calls); 4731 4732 static ssize_t free_calls_show(struct kmem_cache *s, char *buf) 4733 { 4734 if (!(s->flags & SLAB_STORE_USER)) 4735 return -ENOSYS; 4736 return list_locations(s, buf, TRACK_FREE); 4737 } 4738 SLAB_ATTR_RO(free_calls); 4739 #endif /* CONFIG_SLUB_DEBUG */ 4740 4741 #ifdef CONFIG_FAILSLAB 4742 static ssize_t failslab_show(struct kmem_cache *s, char *buf) 4743 { 4744 return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); 4745 } 4746 4747 static ssize_t failslab_store(struct kmem_cache *s, const char *buf, 4748 size_t length) 4749 { 4750 if (s->refcount > 1) 4751 return -EINVAL; 4752 4753 s->flags &= ~SLAB_FAILSLAB; 4754 if (buf[0] == '1') 4755 s->flags |= SLAB_FAILSLAB; 4756 return length; 4757 } 4758 SLAB_ATTR(failslab); 4759 #endif 4760 4761 static ssize_t shrink_show(struct kmem_cache *s, char *buf) 4762 { 4763 return 0; 4764 } 4765 4766 static ssize_t shrink_store(struct kmem_cache *s, 4767 const char *buf, size_t length) 4768 { 4769 if (buf[0] == '1') 4770 kmem_cache_shrink(s); 4771 else 4772 return -EINVAL; 4773 return length; 4774 } 4775 SLAB_ATTR(shrink); 4776 4777 #ifdef CONFIG_NUMA 4778 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) 4779 { 4780 return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); 4781 } 4782 4783 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, 4784 const char *buf, size_t length) 4785 { 4786 unsigned long ratio; 4787 int err; 4788 4789 err = kstrtoul(buf, 10, &ratio); 4790 if (err) 4791 return err; 4792 4793 if (ratio <= 100) 4794 s->remote_node_defrag_ratio = ratio * 10; 4795 4796 return length; 4797 } 4798 SLAB_ATTR(remote_node_defrag_ratio); 4799 #endif 4800 4801 #ifdef CONFIG_SLUB_STATS 4802 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) 4803 { 4804 unsigned long sum = 0; 4805 int cpu; 4806 int len; 4807 int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); 4808 4809 if (!data) 4810 return -ENOMEM; 4811 4812 for_each_online_cpu(cpu) { 4813 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; 4814 4815 data[cpu] = x; 4816 sum += x; 4817 } 4818 4819 len = sprintf(buf, "%lu", sum); 4820 4821 #ifdef CONFIG_SMP 4822 for_each_online_cpu(cpu) { 4823 if (data[cpu] && len < PAGE_SIZE - 20) 4824 len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); 4825 } 4826 #endif 4827 kfree(data); 4828 return len + sprintf(buf + len, "\n"); 4829 } 4830 4831 static void clear_stat(struct kmem_cache *s, enum stat_item si) 4832 { 4833 int cpu; 4834 4835 for_each_online_cpu(cpu) 4836 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; 4837 } 4838 4839 #define STAT_ATTR(si, text) \ 4840 static ssize_t text##_show(struct kmem_cache *s, char *buf) \ 4841 { \ 4842 return show_stat(s, buf, si); \ 4843 } \ 4844 static ssize_t text##_store(struct kmem_cache *s, \ 4845 const char *buf, size_t length) \ 4846 { \ 4847 if (buf[0] != '0') \ 4848 return -EINVAL; \ 4849 clear_stat(s, si); \ 4850 return length; \ 4851 } \ 4852 SLAB_ATTR(text); \ 4853 4854 STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); 4855 STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); 4856 STAT_ATTR(FREE_FASTPATH, free_fastpath); 4857 STAT_ATTR(FREE_SLOWPATH, free_slowpath); 4858 STAT_ATTR(FREE_FROZEN, free_frozen); 4859 STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); 4860 STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); 4861 STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); 4862 STAT_ATTR(ALLOC_SLAB, alloc_slab); 4863 STAT_ATTR(ALLOC_REFILL, alloc_refill); 4864 STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); 4865 STAT_ATTR(FREE_SLAB, free_slab); 4866 STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); 4867 STAT_ATTR(DEACTIVATE_FULL, deactivate_full); 4868 STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); 4869 STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); 4870 STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); 4871 STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); 4872 STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); 4873 STAT_ATTR(ORDER_FALLBACK, order_fallback); 4874 STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); 4875 STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); 4876 STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); 4877 STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); 4878 STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); 4879 STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); 4880 #endif 4881 4882 static struct attribute *slab_attrs[] = { 4883 &slab_size_attr.attr, 4884 &object_size_attr.attr, 4885 &objs_per_slab_attr.attr, 4886 &order_attr.attr, 4887 &min_partial_attr.attr, 4888 &cpu_partial_attr.attr, 4889 &objects_attr.attr, 4890 &objects_partial_attr.attr, 4891 &partial_attr.attr, 4892 &cpu_slabs_attr.attr, 4893 &ctor_attr.attr, 4894 &aliases_attr.attr, 4895 &align_attr.attr, 4896 &hwcache_align_attr.attr, 4897 &reclaim_account_attr.attr, 4898 &destroy_by_rcu_attr.attr, 4899 &shrink_attr.attr, 4900 &reserved_attr.attr, 4901 &slabs_cpu_partial_attr.attr, 4902 #ifdef CONFIG_SLUB_DEBUG 4903 &total_objects_attr.attr, 4904 &slabs_attr.attr, 4905 &sanity_checks_attr.attr, 4906 &trace_attr.attr, 4907 &red_zone_attr.attr, 4908 &poison_attr.attr, 4909 &store_user_attr.attr, 4910 &validate_attr.attr, 4911 &alloc_calls_attr.attr, 4912 &free_calls_attr.attr, 4913 #endif 4914 #ifdef CONFIG_ZONE_DMA 4915 &cache_dma_attr.attr, 4916 #endif 4917 #ifdef CONFIG_NUMA 4918 &remote_node_defrag_ratio_attr.attr, 4919 #endif 4920 #ifdef CONFIG_SLUB_STATS 4921 &alloc_fastpath_attr.attr, 4922 &alloc_slowpath_attr.attr, 4923 &free_fastpath_attr.attr, 4924 &free_slowpath_attr.attr, 4925 &free_frozen_attr.attr, 4926 &free_add_partial_attr.attr, 4927 &free_remove_partial_attr.attr, 4928 &alloc_from_partial_attr.attr, 4929 &alloc_slab_attr.attr, 4930 &alloc_refill_attr.attr, 4931 &alloc_node_mismatch_attr.attr, 4932 &free_slab_attr.attr, 4933 &cpuslab_flush_attr.attr, 4934 &deactivate_full_attr.attr, 4935 &deactivate_empty_attr.attr, 4936 &deactivate_to_head_attr.attr, 4937 &deactivate_to_tail_attr.attr, 4938 &deactivate_remote_frees_attr.attr, 4939 &deactivate_bypass_attr.attr, 4940 &order_fallback_attr.attr, 4941 &cmpxchg_double_fail_attr.attr, 4942 &cmpxchg_double_cpu_fail_attr.attr, 4943 &cpu_partial_alloc_attr.attr, 4944 &cpu_partial_free_attr.attr, 4945 &cpu_partial_node_attr.attr, 4946 &cpu_partial_drain_attr.attr, 4947 #endif 4948 #ifdef CONFIG_FAILSLAB 4949 &failslab_attr.attr, 4950 #endif 4951 4952 NULL 4953 }; 4954 4955 static struct attribute_group slab_attr_group = { 4956 .attrs = slab_attrs, 4957 }; 4958 4959 static ssize_t slab_attr_show(struct kobject *kobj, 4960 struct attribute *attr, 4961 char *buf) 4962 { 4963 struct slab_attribute *attribute; 4964 struct kmem_cache *s; 4965 int err; 4966 4967 attribute = to_slab_attr(attr); 4968 s = to_slab(kobj); 4969 4970 if (!attribute->show) 4971 return -EIO; 4972 4973 err = attribute->show(s, buf); 4974 4975 return err; 4976 } 4977 4978 static ssize_t slab_attr_store(struct kobject *kobj, 4979 struct attribute *attr, 4980 const char *buf, size_t len) 4981 { 4982 struct slab_attribute *attribute; 4983 struct kmem_cache *s; 4984 int err; 4985 4986 attribute = to_slab_attr(attr); 4987 s = to_slab(kobj); 4988 4989 if (!attribute->store) 4990 return -EIO; 4991 4992 err = attribute->store(s, buf, len); 4993 #ifdef CONFIG_MEMCG_KMEM 4994 if (slab_state >= FULL && err >= 0 && is_root_cache(s)) { 4995 struct kmem_cache *c; 4996 4997 mutex_lock(&slab_mutex); 4998 if (s->max_attr_size < len) 4999 s->max_attr_size = len; 5000 5001 /* 5002 * This is a best effort propagation, so this function's return 5003 * value will be determined by the parent cache only. This is 5004 * basically because not all attributes will have a well 5005 * defined semantics for rollbacks - most of the actions will 5006 * have permanent effects. 5007 * 5008 * Returning the error value of any of the children that fail 5009 * is not 100 % defined, in the sense that users seeing the 5010 * error code won't be able to know anything about the state of 5011 * the cache. 5012 * 5013 * Only returning the error code for the parent cache at least 5014 * has well defined semantics. The cache being written to 5015 * directly either failed or succeeded, in which case we loop 5016 * through the descendants with best-effort propagation. 5017 */ 5018 for_each_memcg_cache(c, s) 5019 attribute->store(c, buf, len); 5020 mutex_unlock(&slab_mutex); 5021 } 5022 #endif 5023 return err; 5024 } 5025 5026 static void memcg_propagate_slab_attrs(struct kmem_cache *s) 5027 { 5028 #ifdef CONFIG_MEMCG_KMEM 5029 int i; 5030 char *buffer = NULL; 5031 struct kmem_cache *root_cache; 5032 5033 if (is_root_cache(s)) 5034 return; 5035 5036 root_cache = s->memcg_params.root_cache; 5037 5038 /* 5039 * This mean this cache had no attribute written. Therefore, no point 5040 * in copying default values around 5041 */ 5042 if (!root_cache->max_attr_size) 5043 return; 5044 5045 for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) { 5046 char mbuf[64]; 5047 char *buf; 5048 struct slab_attribute *attr = to_slab_attr(slab_attrs[i]); 5049 5050 if (!attr || !attr->store || !attr->show) 5051 continue; 5052 5053 /* 5054 * It is really bad that we have to allocate here, so we will 5055 * do it only as a fallback. If we actually allocate, though, 5056 * we can just use the allocated buffer until the end. 5057 * 5058 * Most of the slub attributes will tend to be very small in 5059 * size, but sysfs allows buffers up to a page, so they can 5060 * theoretically happen. 5061 */ 5062 if (buffer) 5063 buf = buffer; 5064 else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf)) 5065 buf = mbuf; 5066 else { 5067 buffer = (char *) get_zeroed_page(GFP_KERNEL); 5068 if (WARN_ON(!buffer)) 5069 continue; 5070 buf = buffer; 5071 } 5072 5073 attr->show(root_cache, buf); 5074 attr->store(s, buf, strlen(buf)); 5075 } 5076 5077 if (buffer) 5078 free_page((unsigned long)buffer); 5079 #endif 5080 } 5081 5082 static void kmem_cache_release(struct kobject *k) 5083 { 5084 slab_kmem_cache_release(to_slab(k)); 5085 } 5086 5087 static const struct sysfs_ops slab_sysfs_ops = { 5088 .show = slab_attr_show, 5089 .store = slab_attr_store, 5090 }; 5091 5092 static struct kobj_type slab_ktype = { 5093 .sysfs_ops = &slab_sysfs_ops, 5094 .release = kmem_cache_release, 5095 }; 5096 5097 static int uevent_filter(struct kset *kset, struct kobject *kobj) 5098 { 5099 struct kobj_type *ktype = get_ktype(kobj); 5100 5101 if (ktype == &slab_ktype) 5102 return 1; 5103 return 0; 5104 } 5105 5106 static const struct kset_uevent_ops slab_uevent_ops = { 5107 .filter = uevent_filter, 5108 }; 5109 5110 static struct kset *slab_kset; 5111 5112 static inline struct kset *cache_kset(struct kmem_cache *s) 5113 { 5114 #ifdef CONFIG_MEMCG_KMEM 5115 if (!is_root_cache(s)) 5116 return s->memcg_params.root_cache->memcg_kset; 5117 #endif 5118 return slab_kset; 5119 } 5120 5121 #define ID_STR_LENGTH 64 5122 5123 /* Create a unique string id for a slab cache: 5124 * 5125 * Format :[flags-]size 5126 */ 5127 static char *create_unique_id(struct kmem_cache *s) 5128 { 5129 char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); 5130 char *p = name; 5131 5132 BUG_ON(!name); 5133 5134 *p++ = ':'; 5135 /* 5136 * First flags affecting slabcache operations. We will only 5137 * get here for aliasable slabs so we do not need to support 5138 * too many flags. The flags here must cover all flags that 5139 * are matched during merging to guarantee that the id is 5140 * unique. 5141 */ 5142 if (s->flags & SLAB_CACHE_DMA) 5143 *p++ = 'd'; 5144 if (s->flags & SLAB_RECLAIM_ACCOUNT) 5145 *p++ = 'a'; 5146 if (s->flags & SLAB_DEBUG_FREE) 5147 *p++ = 'F'; 5148 if (!(s->flags & SLAB_NOTRACK)) 5149 *p++ = 't'; 5150 if (p != name + 1) 5151 *p++ = '-'; 5152 p += sprintf(p, "%07d", s->size); 5153 5154 BUG_ON(p > name + ID_STR_LENGTH - 1); 5155 return name; 5156 } 5157 5158 static int sysfs_slab_add(struct kmem_cache *s) 5159 { 5160 int err; 5161 const char *name; 5162 int unmergeable = slab_unmergeable(s); 5163 5164 if (unmergeable) { 5165 /* 5166 * Slabcache can never be merged so we can use the name proper. 5167 * This is typically the case for debug situations. In that 5168 * case we can catch duplicate names easily. 5169 */ 5170 sysfs_remove_link(&slab_kset->kobj, s->name); 5171 name = s->name; 5172 } else { 5173 /* 5174 * Create a unique name for the slab as a target 5175 * for the symlinks. 5176 */ 5177 name = create_unique_id(s); 5178 } 5179 5180 s->kobj.kset = cache_kset(s); 5181 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); 5182 if (err) 5183 goto out_put_kobj; 5184 5185 err = sysfs_create_group(&s->kobj, &slab_attr_group); 5186 if (err) 5187 goto out_del_kobj; 5188 5189 #ifdef CONFIG_MEMCG_KMEM 5190 if (is_root_cache(s)) { 5191 s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); 5192 if (!s->memcg_kset) { 5193 err = -ENOMEM; 5194 goto out_del_kobj; 5195 } 5196 } 5197 #endif 5198 5199 kobject_uevent(&s->kobj, KOBJ_ADD); 5200 if (!unmergeable) { 5201 /* Setup first alias */ 5202 sysfs_slab_alias(s, s->name); 5203 } 5204 out: 5205 if (!unmergeable) 5206 kfree(name); 5207 return err; 5208 out_del_kobj: 5209 kobject_del(&s->kobj); 5210 out_put_kobj: 5211 kobject_put(&s->kobj); 5212 goto out; 5213 } 5214 5215 void sysfs_slab_remove(struct kmem_cache *s) 5216 { 5217 if (slab_state < FULL) 5218 /* 5219 * Sysfs has not been setup yet so no need to remove the 5220 * cache from sysfs. 5221 */ 5222 return; 5223 5224 #ifdef CONFIG_MEMCG_KMEM 5225 kset_unregister(s->memcg_kset); 5226 #endif 5227 kobject_uevent(&s->kobj, KOBJ_REMOVE); 5228 kobject_del(&s->kobj); 5229 kobject_put(&s->kobj); 5230 } 5231 5232 /* 5233 * Need to buffer aliases during bootup until sysfs becomes 5234 * available lest we lose that information. 5235 */ 5236 struct saved_alias { 5237 struct kmem_cache *s; 5238 const char *name; 5239 struct saved_alias *next; 5240 }; 5241 5242 static struct saved_alias *alias_list; 5243 5244 static int sysfs_slab_alias(struct kmem_cache *s, const char *name) 5245 { 5246 struct saved_alias *al; 5247 5248 if (slab_state == FULL) { 5249 /* 5250 * If we have a leftover link then remove it. 5251 */ 5252 sysfs_remove_link(&slab_kset->kobj, name); 5253 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); 5254 } 5255 5256 al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); 5257 if (!al) 5258 return -ENOMEM; 5259 5260 al->s = s; 5261 al->name = name; 5262 al->next = alias_list; 5263 alias_list = al; 5264 return 0; 5265 } 5266 5267 static int __init slab_sysfs_init(void) 5268 { 5269 struct kmem_cache *s; 5270 int err; 5271 5272 mutex_lock(&slab_mutex); 5273 5274 slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); 5275 if (!slab_kset) { 5276 mutex_unlock(&slab_mutex); 5277 pr_err("Cannot register slab subsystem.\n"); 5278 return -ENOSYS; 5279 } 5280 5281 slab_state = FULL; 5282 5283 list_for_each_entry(s, &slab_caches, list) { 5284 err = sysfs_slab_add(s); 5285 if (err) 5286 pr_err("SLUB: Unable to add boot slab %s to sysfs\n", 5287 s->name); 5288 } 5289 5290 while (alias_list) { 5291 struct saved_alias *al = alias_list; 5292 5293 alias_list = alias_list->next; 5294 err = sysfs_slab_alias(al->s, al->name); 5295 if (err) 5296 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", 5297 al->name); 5298 kfree(al); 5299 } 5300 5301 mutex_unlock(&slab_mutex); 5302 resiliency_test(); 5303 return 0; 5304 } 5305 5306 __initcall(slab_sysfs_init); 5307 #endif /* CONFIG_SYSFS */ 5308 5309 /* 5310 * The /proc/slabinfo ABI 5311 */ 5312 #ifdef CONFIG_SLABINFO 5313 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) 5314 { 5315 unsigned long nr_slabs = 0; 5316 unsigned long nr_objs = 0; 5317 unsigned long nr_free = 0; 5318 int node; 5319 struct kmem_cache_node *n; 5320 5321 for_each_kmem_cache_node(s, node, n) { 5322 nr_slabs += node_nr_slabs(n); 5323 nr_objs += node_nr_objs(n); 5324 nr_free += count_partial(n, count_free); 5325 } 5326 5327 sinfo->active_objs = nr_objs - nr_free; 5328 sinfo->num_objs = nr_objs; 5329 sinfo->active_slabs = nr_slabs; 5330 sinfo->num_slabs = nr_slabs; 5331 sinfo->objects_per_slab = oo_objects(s->oo); 5332 sinfo->cache_order = oo_order(s->oo); 5333 } 5334 5335 void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s) 5336 { 5337 } 5338 5339 ssize_t slabinfo_write(struct file *file, const char __user *buffer, 5340 size_t count, loff_t *ppos) 5341 { 5342 return -EIO; 5343 } 5344 #endif /* CONFIG_SLABINFO */ 5345