1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 28fa3ed80SDennis Zhou #ifndef _MM_PERCPU_INTERNAL_H 38fa3ed80SDennis Zhou #define _MM_PERCPU_INTERNAL_H 48fa3ed80SDennis Zhou 58fa3ed80SDennis Zhou #include <linux/types.h> 68fa3ed80SDennis Zhou #include <linux/percpu.h> 78fa3ed80SDennis Zhou 8ca460b3cSDennis Zhou (Facebook) /* 93c7be18aSRoman Gushchin * There are two chunk types: root and memcg-aware. 103c7be18aSRoman Gushchin * Chunks of each type have separate slots list. 113c7be18aSRoman Gushchin * 123c7be18aSRoman Gushchin * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is 133c7be18aSRoman Gushchin * used to store memcg membership data of a percpu object. Obj_cgroups are 143c7be18aSRoman Gushchin * ref-counted pointers to a memory cgroup with an ability to switch dynamically 153c7be18aSRoman Gushchin * to the parent memory cgroup. This allows to reclaim a deleted memory cgroup 163c7be18aSRoman Gushchin * without reclaiming of all outstanding objects, which hold a reference at it. 173c7be18aSRoman Gushchin */ 183c7be18aSRoman Gushchin enum pcpu_chunk_type { 193c7be18aSRoman Gushchin PCPU_CHUNK_ROOT, 203c7be18aSRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 213c7be18aSRoman Gushchin PCPU_CHUNK_MEMCG, 223c7be18aSRoman Gushchin #endif 233c7be18aSRoman Gushchin PCPU_NR_CHUNK_TYPES, 243c7be18aSRoman Gushchin PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES 253c7be18aSRoman Gushchin }; 263c7be18aSRoman Gushchin 273c7be18aSRoman Gushchin /* 28ca460b3cSDennis Zhou (Facebook) * pcpu_block_md is the metadata block struct. 29ca460b3cSDennis Zhou (Facebook) * Each chunk's bitmap is split into a number of full blocks. 30ca460b3cSDennis Zhou (Facebook) * All units are in terms of bits. 31382b88e9SDennis Zhou * 32382b88e9SDennis Zhou * The scan hint is the largest known contiguous area before the contig hint. 33382b88e9SDennis Zhou * It is not necessarily the actual largest contig hint though. There is an 34382b88e9SDennis Zhou * invariant that the scan_hint_start > contig_hint_start iff 35382b88e9SDennis Zhou * scan_hint == contig_hint. This is necessary because when scanning forward, 36382b88e9SDennis Zhou * we don't know if a new contig hint would be better than the current one. 37ca460b3cSDennis Zhou (Facebook) */ 38ca460b3cSDennis Zhou (Facebook) struct pcpu_block_md { 39382b88e9SDennis Zhou int scan_hint; /* scan hint for block */ 40382b88e9SDennis Zhou int scan_hint_start; /* block relative starting 41382b88e9SDennis Zhou position of the scan hint */ 42ca460b3cSDennis Zhou (Facebook) int contig_hint; /* contig hint for block */ 43ca460b3cSDennis Zhou (Facebook) int contig_hint_start; /* block relative starting 44ca460b3cSDennis Zhou (Facebook) position of the contig hint */ 45ca460b3cSDennis Zhou (Facebook) int left_free; /* size of free space along 46ca460b3cSDennis Zhou (Facebook) the left side of the block */ 47ca460b3cSDennis Zhou (Facebook) int right_free; /* size of free space along 48ca460b3cSDennis Zhou (Facebook) the right side of the block */ 49ca460b3cSDennis Zhou (Facebook) int first_free; /* block position of first free */ 50047924c9SDennis Zhou int nr_bits; /* total bits responsible for */ 51ca460b3cSDennis Zhou (Facebook) }; 52ca460b3cSDennis Zhou (Facebook) 538fa3ed80SDennis Zhou struct pcpu_chunk { 5430a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 5530a5b536SDennis Zhou int nr_alloc; /* # of allocations */ 5630a5b536SDennis Zhou size_t max_alloc_size; /* largest allocation size */ 5730a5b536SDennis Zhou #endif 5830a5b536SDennis Zhou 598fa3ed80SDennis Zhou struct list_head list; /* linked to pcpu_slot lists */ 6040064aecSDennis Zhou (Facebook) int free_bytes; /* free bytes in the chunk */ 6192c14cabSDennis Zhou struct pcpu_block_md chunk_md; 628fa3ed80SDennis Zhou void *base_addr; /* base address of this chunk */ 638fa3ed80SDennis Zhou 6440064aecSDennis Zhou (Facebook) unsigned long *alloc_map; /* allocation map */ 6540064aecSDennis Zhou (Facebook) unsigned long *bound_map; /* boundary map */ 66ca460b3cSDennis Zhou (Facebook) struct pcpu_block_md *md_blocks; /* metadata blocks */ 678fa3ed80SDennis Zhou 688fa3ed80SDennis Zhou void *data; /* chunk data */ 698fa3ed80SDennis Zhou bool immutable; /* no [de]population allowed */ 70*f1833241SRoman Gushchin bool isolated; /* isolated from active chunk 71*f1833241SRoman Gushchin slots */ 72e2266705SDennis Zhou (Facebook) int start_offset; /* the overlap with the previous 73e2266705SDennis Zhou (Facebook) region to have a page aligned 74e2266705SDennis Zhou (Facebook) base_addr */ 756b9d7c8eSDennis Zhou (Facebook) int end_offset; /* additional area required to 766b9d7c8eSDennis Zhou (Facebook) have the region end page 776b9d7c8eSDennis Zhou (Facebook) aligned */ 783c7be18aSRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 793c7be18aSRoman Gushchin struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 803c7be18aSRoman Gushchin #endif 81c0ebfdc3SDennis Zhou (Facebook) 82c0ebfdc3SDennis Zhou (Facebook) int nr_pages; /* # of pages served by this chunk */ 838fa3ed80SDennis Zhou int nr_populated; /* # of populated pages */ 840cecf50cSDennis Zhou (Facebook) int nr_empty_pop_pages; /* # of empty populated pages */ 858fa3ed80SDennis Zhou unsigned long populated[]; /* populated bitmap */ 868fa3ed80SDennis Zhou }; 878fa3ed80SDennis Zhou 888fa3ed80SDennis Zhou extern spinlock_t pcpu_lock; 898fa3ed80SDennis Zhou 903c7be18aSRoman Gushchin extern struct list_head *pcpu_chunk_lists; 918fa3ed80SDennis Zhou extern int pcpu_nr_slots; 92*f1833241SRoman Gushchin extern int pcpu_sidelined_slot; 93*f1833241SRoman Gushchin extern int pcpu_to_depopulate_slot; 940760fa3dSRoman Gushchin extern int pcpu_nr_empty_pop_pages[]; 958fa3ed80SDennis Zhou 968fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_first_chunk; 978fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_reserved_chunk; 988fa3ed80SDennis Zhou 9940064aecSDennis Zhou (Facebook) /** 100ca460b3cSDennis Zhou (Facebook) * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 101ca460b3cSDennis Zhou (Facebook) * @chunk: chunk of interest 102ca460b3cSDennis Zhou (Facebook) * 103ca460b3cSDennis Zhou (Facebook) * This conversion is from the number of physical pages that the chunk 104ca460b3cSDennis Zhou (Facebook) * serves to the number of bitmap blocks used. 105ca460b3cSDennis Zhou (Facebook) */ 106ca460b3cSDennis Zhou (Facebook) static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 107ca460b3cSDennis Zhou (Facebook) { 108ca460b3cSDennis Zhou (Facebook) return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 109ca460b3cSDennis Zhou (Facebook) } 110ca460b3cSDennis Zhou (Facebook) 111ca460b3cSDennis Zhou (Facebook) /** 11240064aecSDennis Zhou (Facebook) * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 11340064aecSDennis Zhou (Facebook) * @pages: number of physical pages 11440064aecSDennis Zhou (Facebook) * 11540064aecSDennis Zhou (Facebook) * This conversion is from physical pages to the number of bits 11640064aecSDennis Zhou (Facebook) * required in the bitmap. 11740064aecSDennis Zhou (Facebook) */ 11840064aecSDennis Zhou (Facebook) static inline int pcpu_nr_pages_to_map_bits(int pages) 11940064aecSDennis Zhou (Facebook) { 12040064aecSDennis Zhou (Facebook) return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 12140064aecSDennis Zhou (Facebook) } 12240064aecSDennis Zhou (Facebook) 12340064aecSDennis Zhou (Facebook) /** 12440064aecSDennis Zhou (Facebook) * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 12540064aecSDennis Zhou (Facebook) * @chunk: chunk of interest 12640064aecSDennis Zhou (Facebook) * 12740064aecSDennis Zhou (Facebook) * This conversion is from the number of physical pages that the chunk 12840064aecSDennis Zhou (Facebook) * serves to the number of bits in the bitmap. 12940064aecSDennis Zhou (Facebook) */ 13040064aecSDennis Zhou (Facebook) static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 13140064aecSDennis Zhou (Facebook) { 13240064aecSDennis Zhou (Facebook) return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 13340064aecSDennis Zhou (Facebook) } 13440064aecSDennis Zhou (Facebook) 1353c7be18aSRoman Gushchin #ifdef CONFIG_MEMCG_KMEM 1363c7be18aSRoman Gushchin static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk) 1373c7be18aSRoman Gushchin { 1383c7be18aSRoman Gushchin if (chunk->obj_cgroups) 1393c7be18aSRoman Gushchin return PCPU_CHUNK_MEMCG; 1403c7be18aSRoman Gushchin return PCPU_CHUNK_ROOT; 1413c7be18aSRoman Gushchin } 1423c7be18aSRoman Gushchin 1433c7be18aSRoman Gushchin static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type) 1443c7be18aSRoman Gushchin { 1453c7be18aSRoman Gushchin return chunk_type == PCPU_CHUNK_MEMCG; 1463c7be18aSRoman Gushchin } 1473c7be18aSRoman Gushchin 1483c7be18aSRoman Gushchin #else 1493c7be18aSRoman Gushchin static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk) 1503c7be18aSRoman Gushchin { 1513c7be18aSRoman Gushchin return PCPU_CHUNK_ROOT; 1523c7be18aSRoman Gushchin } 1533c7be18aSRoman Gushchin 1543c7be18aSRoman Gushchin static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type) 1553c7be18aSRoman Gushchin { 1563c7be18aSRoman Gushchin return false; 1573c7be18aSRoman Gushchin } 1583c7be18aSRoman Gushchin #endif 1593c7be18aSRoman Gushchin 1603c7be18aSRoman Gushchin static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type) 1613c7be18aSRoman Gushchin { 1623c7be18aSRoman Gushchin return &pcpu_chunk_lists[pcpu_nr_slots * 1633c7be18aSRoman Gushchin pcpu_is_memcg_chunk(chunk_type)]; 1643c7be18aSRoman Gushchin } 1653c7be18aSRoman Gushchin 16630a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 16730a5b536SDennis Zhou 16830a5b536SDennis Zhou #include <linux/spinlock.h> 16930a5b536SDennis Zhou 17030a5b536SDennis Zhou struct percpu_stats { 17130a5b536SDennis Zhou u64 nr_alloc; /* lifetime # of allocations */ 17230a5b536SDennis Zhou u64 nr_dealloc; /* lifetime # of deallocations */ 17330a5b536SDennis Zhou u64 nr_cur_alloc; /* current # of allocations */ 17430a5b536SDennis Zhou u64 nr_max_alloc; /* max # of live allocations */ 17530a5b536SDennis Zhou u32 nr_chunks; /* current # of live chunks */ 17630a5b536SDennis Zhou u32 nr_max_chunks; /* max # of live chunks */ 17730a5b536SDennis Zhou size_t min_alloc_size; /* min allocaiton size */ 17830a5b536SDennis Zhou size_t max_alloc_size; /* max allocation size */ 17930a5b536SDennis Zhou }; 18030a5b536SDennis Zhou 18130a5b536SDennis Zhou extern struct percpu_stats pcpu_stats; 18230a5b536SDennis Zhou extern struct pcpu_alloc_info pcpu_stats_ai; 18330a5b536SDennis Zhou 18430a5b536SDennis Zhou /* 18530a5b536SDennis Zhou * For debug purposes. We don't care about the flexible array. 18630a5b536SDennis Zhou */ 18730a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 18830a5b536SDennis Zhou { 18930a5b536SDennis Zhou memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 19030a5b536SDennis Zhou 19130a5b536SDennis Zhou /* initialize min_alloc_size to unit_size */ 19230a5b536SDennis Zhou pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 19330a5b536SDennis Zhou } 19430a5b536SDennis Zhou 19530a5b536SDennis Zhou /* 19630a5b536SDennis Zhou * pcpu_stats_area_alloc - increment area allocation stats 19730a5b536SDennis Zhou * @chunk: the location of the area being allocated 19830a5b536SDennis Zhou * @size: size of area to allocate in bytes 19930a5b536SDennis Zhou * 20030a5b536SDennis Zhou * CONTEXT: 20130a5b536SDennis Zhou * pcpu_lock. 20230a5b536SDennis Zhou */ 20330a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 20430a5b536SDennis Zhou { 20530a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 20630a5b536SDennis Zhou 20730a5b536SDennis Zhou pcpu_stats.nr_alloc++; 20830a5b536SDennis Zhou pcpu_stats.nr_cur_alloc++; 20930a5b536SDennis Zhou pcpu_stats.nr_max_alloc = 21030a5b536SDennis Zhou max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 21130a5b536SDennis Zhou pcpu_stats.min_alloc_size = 21230a5b536SDennis Zhou min(pcpu_stats.min_alloc_size, size); 21330a5b536SDennis Zhou pcpu_stats.max_alloc_size = 21430a5b536SDennis Zhou max(pcpu_stats.max_alloc_size, size); 21530a5b536SDennis Zhou 21630a5b536SDennis Zhou chunk->nr_alloc++; 21730a5b536SDennis Zhou chunk->max_alloc_size = max(chunk->max_alloc_size, size); 21830a5b536SDennis Zhou } 21930a5b536SDennis Zhou 22030a5b536SDennis Zhou /* 22130a5b536SDennis Zhou * pcpu_stats_area_dealloc - decrement allocation stats 22230a5b536SDennis Zhou * @chunk: the location of the area being deallocated 22330a5b536SDennis Zhou * 22430a5b536SDennis Zhou * CONTEXT: 22530a5b536SDennis Zhou * pcpu_lock. 22630a5b536SDennis Zhou */ 22730a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 22830a5b536SDennis Zhou { 22930a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 23030a5b536SDennis Zhou 23130a5b536SDennis Zhou pcpu_stats.nr_dealloc++; 23230a5b536SDennis Zhou pcpu_stats.nr_cur_alloc--; 23330a5b536SDennis Zhou 23430a5b536SDennis Zhou chunk->nr_alloc--; 23530a5b536SDennis Zhou } 23630a5b536SDennis Zhou 23730a5b536SDennis Zhou /* 23830a5b536SDennis Zhou * pcpu_stats_chunk_alloc - increment chunk stats 23930a5b536SDennis Zhou */ 24030a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 24130a5b536SDennis Zhou { 242303abfdfSDennis Zhou unsigned long flags; 243303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 24430a5b536SDennis Zhou 24530a5b536SDennis Zhou pcpu_stats.nr_chunks++; 24630a5b536SDennis Zhou pcpu_stats.nr_max_chunks = 24730a5b536SDennis Zhou max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 24830a5b536SDennis Zhou 249303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 25030a5b536SDennis Zhou } 25130a5b536SDennis Zhou 25230a5b536SDennis Zhou /* 25330a5b536SDennis Zhou * pcpu_stats_chunk_dealloc - decrement chunk stats 25430a5b536SDennis Zhou */ 25530a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 25630a5b536SDennis Zhou { 257303abfdfSDennis Zhou unsigned long flags; 258303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 25930a5b536SDennis Zhou 26030a5b536SDennis Zhou pcpu_stats.nr_chunks--; 26130a5b536SDennis Zhou 262303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 26330a5b536SDennis Zhou } 26430a5b536SDennis Zhou 26530a5b536SDennis Zhou #else 26630a5b536SDennis Zhou 26730a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 26830a5b536SDennis Zhou { 26930a5b536SDennis Zhou } 27030a5b536SDennis Zhou 27130a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 27230a5b536SDennis Zhou { 27330a5b536SDennis Zhou } 27430a5b536SDennis Zhou 27530a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 27630a5b536SDennis Zhou { 27730a5b536SDennis Zhou } 27830a5b536SDennis Zhou 27930a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 28030a5b536SDennis Zhou { 28130a5b536SDennis Zhou } 28230a5b536SDennis Zhou 28330a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 28430a5b536SDennis Zhou { 28530a5b536SDennis Zhou } 28630a5b536SDennis Zhou 28730a5b536SDennis Zhou #endif /* !CONFIG_PERCPU_STATS */ 28830a5b536SDennis Zhou 2898fa3ed80SDennis Zhou #endif 290