1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 8 /* 9 * pcpu_block_md is the metadata block struct. 10 * Each chunk's bitmap is split into a number of full blocks. 11 * All units are in terms of bits. 12 * 13 * The scan hint is the largest known contiguous area before the contig hint. 14 * It is not necessarily the actual largest contig hint though. There is an 15 * invariant that the scan_hint_start > contig_hint_start iff 16 * scan_hint == contig_hint. This is necessary because when scanning forward, 17 * we don't know if a new contig hint would be better than the current one. 18 */ 19 struct pcpu_block_md { 20 int scan_hint; /* scan hint for block */ 21 int scan_hint_start; /* block relative starting 22 position of the scan hint */ 23 int contig_hint; /* contig hint for block */ 24 int contig_hint_start; /* block relative starting 25 position of the contig hint */ 26 int left_free; /* size of free space along 27 the left side of the block */ 28 int right_free; /* size of free space along 29 the right side of the block */ 30 int first_free; /* block position of first free */ 31 int nr_bits; /* total bits responsible for */ 32 }; 33 34 struct pcpu_chunk { 35 #ifdef CONFIG_PERCPU_STATS 36 int nr_alloc; /* # of allocations */ 37 size_t max_alloc_size; /* largest allocation size */ 38 #endif 39 40 struct list_head list; /* linked to pcpu_slot lists */ 41 int free_bytes; /* free bytes in the chunk */ 42 struct pcpu_block_md chunk_md; 43 void *base_addr; /* base address of this chunk */ 44 45 unsigned long *alloc_map; /* allocation map */ 46 unsigned long *bound_map; /* boundary map */ 47 struct pcpu_block_md *md_blocks; /* metadata blocks */ 48 49 void *data; /* chunk data */ 50 bool immutable; /* no [de]population allowed */ 51 bool isolated; /* isolated from active chunk 52 slots */ 53 int start_offset; /* the overlap with the previous 54 region to have a page aligned 55 base_addr */ 56 int end_offset; /* additional area required to 57 have the region end page 58 aligned */ 59 #ifdef CONFIG_MEMCG_KMEM 60 struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 61 #endif 62 63 int nr_pages; /* # of pages served by this chunk */ 64 int nr_populated; /* # of populated pages */ 65 int nr_empty_pop_pages; /* # of empty populated pages */ 66 unsigned long populated[]; /* populated bitmap */ 67 }; 68 69 extern spinlock_t pcpu_lock; 70 71 extern struct list_head *pcpu_chunk_lists; 72 extern int pcpu_nr_slots; 73 extern int pcpu_sidelined_slot; 74 extern int pcpu_to_depopulate_slot; 75 extern int pcpu_nr_empty_pop_pages; 76 77 extern struct pcpu_chunk *pcpu_first_chunk; 78 extern struct pcpu_chunk *pcpu_reserved_chunk; 79 80 /** 81 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 82 * @chunk: chunk of interest 83 * 84 * This conversion is from the number of physical pages that the chunk 85 * serves to the number of bitmap blocks used. 86 */ 87 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 88 { 89 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 90 } 91 92 /** 93 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 94 * @pages: number of physical pages 95 * 96 * This conversion is from physical pages to the number of bits 97 * required in the bitmap. 98 */ 99 static inline int pcpu_nr_pages_to_map_bits(int pages) 100 { 101 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 102 } 103 104 /** 105 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 106 * @chunk: chunk of interest 107 * 108 * This conversion is from the number of physical pages that the chunk 109 * serves to the number of bits in the bitmap. 110 */ 111 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 112 { 113 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 114 } 115 116 #ifdef CONFIG_PERCPU_STATS 117 118 #include <linux/spinlock.h> 119 120 struct percpu_stats { 121 u64 nr_alloc; /* lifetime # of allocations */ 122 u64 nr_dealloc; /* lifetime # of deallocations */ 123 u64 nr_cur_alloc; /* current # of allocations */ 124 u64 nr_max_alloc; /* max # of live allocations */ 125 u32 nr_chunks; /* current # of live chunks */ 126 u32 nr_max_chunks; /* max # of live chunks */ 127 size_t min_alloc_size; /* min allocation size */ 128 size_t max_alloc_size; /* max allocation size */ 129 }; 130 131 extern struct percpu_stats pcpu_stats; 132 extern struct pcpu_alloc_info pcpu_stats_ai; 133 134 /* 135 * For debug purposes. We don't care about the flexible array. 136 */ 137 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 138 { 139 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 140 141 /* initialize min_alloc_size to unit_size */ 142 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 143 } 144 145 /* 146 * pcpu_stats_area_alloc - increment area allocation stats 147 * @chunk: the location of the area being allocated 148 * @size: size of area to allocate in bytes 149 * 150 * CONTEXT: 151 * pcpu_lock. 152 */ 153 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 154 { 155 lockdep_assert_held(&pcpu_lock); 156 157 pcpu_stats.nr_alloc++; 158 pcpu_stats.nr_cur_alloc++; 159 pcpu_stats.nr_max_alloc = 160 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 161 pcpu_stats.min_alloc_size = 162 min(pcpu_stats.min_alloc_size, size); 163 pcpu_stats.max_alloc_size = 164 max(pcpu_stats.max_alloc_size, size); 165 166 chunk->nr_alloc++; 167 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 168 } 169 170 /* 171 * pcpu_stats_area_dealloc - decrement allocation stats 172 * @chunk: the location of the area being deallocated 173 * 174 * CONTEXT: 175 * pcpu_lock. 176 */ 177 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 178 { 179 lockdep_assert_held(&pcpu_lock); 180 181 pcpu_stats.nr_dealloc++; 182 pcpu_stats.nr_cur_alloc--; 183 184 chunk->nr_alloc--; 185 } 186 187 /* 188 * pcpu_stats_chunk_alloc - increment chunk stats 189 */ 190 static inline void pcpu_stats_chunk_alloc(void) 191 { 192 unsigned long flags; 193 spin_lock_irqsave(&pcpu_lock, flags); 194 195 pcpu_stats.nr_chunks++; 196 pcpu_stats.nr_max_chunks = 197 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 198 199 spin_unlock_irqrestore(&pcpu_lock, flags); 200 } 201 202 /* 203 * pcpu_stats_chunk_dealloc - decrement chunk stats 204 */ 205 static inline void pcpu_stats_chunk_dealloc(void) 206 { 207 unsigned long flags; 208 spin_lock_irqsave(&pcpu_lock, flags); 209 210 pcpu_stats.nr_chunks--; 211 212 spin_unlock_irqrestore(&pcpu_lock, flags); 213 } 214 215 #else 216 217 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 218 { 219 } 220 221 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 222 { 223 } 224 225 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 226 { 227 } 228 229 static inline void pcpu_stats_chunk_alloc(void) 230 { 231 } 232 233 static inline void pcpu_stats_chunk_dealloc(void) 234 { 235 } 236 237 #endif /* !CONFIG_PERCPU_STATS */ 238 239 #endif 240