1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 8 /* 9 * There are two chunk types: root and memcg-aware. 10 * Chunks of each type have separate slots list. 11 * 12 * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is 13 * used to store memcg membership data of a percpu object. Obj_cgroups are 14 * ref-counted pointers to a memory cgroup with an ability to switch dynamically 15 * to the parent memory cgroup. This allows to reclaim a deleted memory cgroup 16 * without reclaiming of all outstanding objects, which hold a reference at it. 17 */ 18 enum pcpu_chunk_type { 19 PCPU_CHUNK_ROOT, 20 #ifdef CONFIG_MEMCG_KMEM 21 PCPU_CHUNK_MEMCG, 22 #endif 23 PCPU_NR_CHUNK_TYPES, 24 PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES 25 }; 26 27 /* 28 * pcpu_block_md is the metadata block struct. 29 * Each chunk's bitmap is split into a number of full blocks. 30 * All units are in terms of bits. 31 * 32 * The scan hint is the largest known contiguous area before the contig hint. 33 * It is not necessarily the actual largest contig hint though. There is an 34 * invariant that the scan_hint_start > contig_hint_start iff 35 * scan_hint == contig_hint. This is necessary because when scanning forward, 36 * we don't know if a new contig hint would be better than the current one. 37 */ 38 struct pcpu_block_md { 39 int scan_hint; /* scan hint for block */ 40 int scan_hint_start; /* block relative starting 41 position of the scan hint */ 42 int contig_hint; /* contig hint for block */ 43 int contig_hint_start; /* block relative starting 44 position of the contig hint */ 45 int left_free; /* size of free space along 46 the left side of the block */ 47 int right_free; /* size of free space along 48 the right side of the block */ 49 int first_free; /* block position of first free */ 50 int nr_bits; /* total bits responsible for */ 51 }; 52 53 struct pcpu_chunk { 54 #ifdef CONFIG_PERCPU_STATS 55 int nr_alloc; /* # of allocations */ 56 size_t max_alloc_size; /* largest allocation size */ 57 #endif 58 59 struct list_head list; /* linked to pcpu_slot lists */ 60 int free_bytes; /* free bytes in the chunk */ 61 struct pcpu_block_md chunk_md; 62 void *base_addr; /* base address of this chunk */ 63 64 unsigned long *alloc_map; /* allocation map */ 65 unsigned long *bound_map; /* boundary map */ 66 struct pcpu_block_md *md_blocks; /* metadata blocks */ 67 68 void *data; /* chunk data */ 69 bool immutable; /* no [de]population allowed */ 70 int start_offset; /* the overlap with the previous 71 region to have a page aligned 72 base_addr */ 73 int end_offset; /* additional area required to 74 have the region end page 75 aligned */ 76 #ifdef CONFIG_MEMCG_KMEM 77 struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 78 #endif 79 80 int nr_pages; /* # of pages served by this chunk */ 81 int nr_populated; /* # of populated pages */ 82 int nr_empty_pop_pages; /* # of empty populated pages */ 83 unsigned long populated[]; /* populated bitmap */ 84 }; 85 86 extern spinlock_t pcpu_lock; 87 88 extern struct list_head *pcpu_chunk_lists; 89 extern int pcpu_nr_slots; 90 extern int pcpu_nr_empty_pop_pages[]; 91 92 extern struct pcpu_chunk *pcpu_first_chunk; 93 extern struct pcpu_chunk *pcpu_reserved_chunk; 94 95 /** 96 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 97 * @chunk: chunk of interest 98 * 99 * This conversion is from the number of physical pages that the chunk 100 * serves to the number of bitmap blocks used. 101 */ 102 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 103 { 104 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 105 } 106 107 /** 108 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 109 * @pages: number of physical pages 110 * 111 * This conversion is from physical pages to the number of bits 112 * required in the bitmap. 113 */ 114 static inline int pcpu_nr_pages_to_map_bits(int pages) 115 { 116 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 117 } 118 119 /** 120 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 121 * @chunk: chunk of interest 122 * 123 * This conversion is from the number of physical pages that the chunk 124 * serves to the number of bits in the bitmap. 125 */ 126 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 127 { 128 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 129 } 130 131 #ifdef CONFIG_MEMCG_KMEM 132 static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk) 133 { 134 if (chunk->obj_cgroups) 135 return PCPU_CHUNK_MEMCG; 136 return PCPU_CHUNK_ROOT; 137 } 138 139 static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type) 140 { 141 return chunk_type == PCPU_CHUNK_MEMCG; 142 } 143 144 #else 145 static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk) 146 { 147 return PCPU_CHUNK_ROOT; 148 } 149 150 static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type) 151 { 152 return false; 153 } 154 #endif 155 156 static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type) 157 { 158 return &pcpu_chunk_lists[pcpu_nr_slots * 159 pcpu_is_memcg_chunk(chunk_type)]; 160 } 161 162 #ifdef CONFIG_PERCPU_STATS 163 164 #include <linux/spinlock.h> 165 166 struct percpu_stats { 167 u64 nr_alloc; /* lifetime # of allocations */ 168 u64 nr_dealloc; /* lifetime # of deallocations */ 169 u64 nr_cur_alloc; /* current # of allocations */ 170 u64 nr_max_alloc; /* max # of live allocations */ 171 u32 nr_chunks; /* current # of live chunks */ 172 u32 nr_max_chunks; /* max # of live chunks */ 173 size_t min_alloc_size; /* min allocation size */ 174 size_t max_alloc_size; /* max allocation size */ 175 }; 176 177 extern struct percpu_stats pcpu_stats; 178 extern struct pcpu_alloc_info pcpu_stats_ai; 179 180 /* 181 * For debug purposes. We don't care about the flexible array. 182 */ 183 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 184 { 185 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 186 187 /* initialize min_alloc_size to unit_size */ 188 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 189 } 190 191 /* 192 * pcpu_stats_area_alloc - increment area allocation stats 193 * @chunk: the location of the area being allocated 194 * @size: size of area to allocate in bytes 195 * 196 * CONTEXT: 197 * pcpu_lock. 198 */ 199 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 200 { 201 lockdep_assert_held(&pcpu_lock); 202 203 pcpu_stats.nr_alloc++; 204 pcpu_stats.nr_cur_alloc++; 205 pcpu_stats.nr_max_alloc = 206 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 207 pcpu_stats.min_alloc_size = 208 min(pcpu_stats.min_alloc_size, size); 209 pcpu_stats.max_alloc_size = 210 max(pcpu_stats.max_alloc_size, size); 211 212 chunk->nr_alloc++; 213 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 214 } 215 216 /* 217 * pcpu_stats_area_dealloc - decrement allocation stats 218 * @chunk: the location of the area being deallocated 219 * 220 * CONTEXT: 221 * pcpu_lock. 222 */ 223 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 224 { 225 lockdep_assert_held(&pcpu_lock); 226 227 pcpu_stats.nr_dealloc++; 228 pcpu_stats.nr_cur_alloc--; 229 230 chunk->nr_alloc--; 231 } 232 233 /* 234 * pcpu_stats_chunk_alloc - increment chunk stats 235 */ 236 static inline void pcpu_stats_chunk_alloc(void) 237 { 238 unsigned long flags; 239 spin_lock_irqsave(&pcpu_lock, flags); 240 241 pcpu_stats.nr_chunks++; 242 pcpu_stats.nr_max_chunks = 243 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 244 245 spin_unlock_irqrestore(&pcpu_lock, flags); 246 } 247 248 /* 249 * pcpu_stats_chunk_dealloc - decrement chunk stats 250 */ 251 static inline void pcpu_stats_chunk_dealloc(void) 252 { 253 unsigned long flags; 254 spin_lock_irqsave(&pcpu_lock, flags); 255 256 pcpu_stats.nr_chunks--; 257 258 spin_unlock_irqrestore(&pcpu_lock, flags); 259 } 260 261 #else 262 263 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 264 { 265 } 266 267 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 268 { 269 } 270 271 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 272 { 273 } 274 275 static inline void pcpu_stats_chunk_alloc(void) 276 { 277 } 278 279 static inline void pcpu_stats_chunk_dealloc(void) 280 { 281 } 282 283 #endif /* !CONFIG_PERCPU_STATS */ 284 285 #endif 286