1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 #include <linux/memcontrol.h> 8 9 /* 10 * pcpu_block_md is the metadata block struct. 11 * Each chunk's bitmap is split into a number of full blocks. 12 * All units are in terms of bits. 13 * 14 * The scan hint is the largest known contiguous area before the contig hint. 15 * It is not necessarily the actual largest contig hint though. There is an 16 * invariant that the scan_hint_start > contig_hint_start iff 17 * scan_hint == contig_hint. This is necessary because when scanning forward, 18 * we don't know if a new contig hint would be better than the current one. 19 */ 20 struct pcpu_block_md { 21 int scan_hint; /* scan hint for block */ 22 int scan_hint_start; /* block relative starting 23 position of the scan hint */ 24 int contig_hint; /* contig hint for block */ 25 int contig_hint_start; /* block relative starting 26 position of the contig hint */ 27 int left_free; /* size of free space along 28 the left side of the block */ 29 int right_free; /* size of free space along 30 the right side of the block */ 31 int first_free; /* block position of first free */ 32 int nr_bits; /* total bits responsible for */ 33 }; 34 35 struct pcpu_chunk { 36 #ifdef CONFIG_PERCPU_STATS 37 int nr_alloc; /* # of allocations */ 38 size_t max_alloc_size; /* largest allocation size */ 39 #endif 40 41 struct list_head list; /* linked to pcpu_slot lists */ 42 int free_bytes; /* free bytes in the chunk */ 43 struct pcpu_block_md chunk_md; 44 unsigned long *bound_map; /* boundary map */ 45 46 /* 47 * base_addr is the base address of this chunk. 48 * To reduce false sharing, current layout is optimized to make sure 49 * base_addr locate in the different cacheline with free_bytes and 50 * chunk_md. 51 */ 52 void *base_addr ____cacheline_aligned_in_smp; 53 54 unsigned long *alloc_map; /* allocation map */ 55 struct pcpu_block_md *md_blocks; /* metadata blocks */ 56 57 void *data; /* chunk data */ 58 bool immutable; /* no [de]population allowed */ 59 bool isolated; /* isolated from active chunk 60 slots */ 61 int start_offset; /* the overlap with the previous 62 region to have a page aligned 63 base_addr */ 64 int end_offset; /* additional area required to 65 have the region end page 66 aligned */ 67 #ifdef CONFIG_MEMCG_KMEM 68 struct obj_cgroup **obj_cgroups; /* vector of object cgroups */ 69 #endif 70 71 int nr_pages; /* # of pages served by this chunk */ 72 int nr_populated; /* # of populated pages */ 73 int nr_empty_pop_pages; /* # of empty populated pages */ 74 unsigned long populated[]; /* populated bitmap */ 75 }; 76 77 extern spinlock_t pcpu_lock; 78 79 extern struct list_head *pcpu_chunk_lists; 80 extern int pcpu_nr_slots; 81 extern int pcpu_sidelined_slot; 82 extern int pcpu_to_depopulate_slot; 83 extern int pcpu_nr_empty_pop_pages; 84 85 extern struct pcpu_chunk *pcpu_first_chunk; 86 extern struct pcpu_chunk *pcpu_reserved_chunk; 87 88 /** 89 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 90 * @chunk: chunk of interest 91 * 92 * This conversion is from the number of physical pages that the chunk 93 * serves to the number of bitmap blocks used. 94 */ 95 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 96 { 97 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 98 } 99 100 /** 101 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 102 * @pages: number of physical pages 103 * 104 * This conversion is from physical pages to the number of bits 105 * required in the bitmap. 106 */ 107 static inline int pcpu_nr_pages_to_map_bits(int pages) 108 { 109 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 110 } 111 112 /** 113 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 114 * @chunk: chunk of interest 115 * 116 * This conversion is from the number of physical pages that the chunk 117 * serves to the number of bits in the bitmap. 118 */ 119 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 120 { 121 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 122 } 123 124 /** 125 * pcpu_obj_full_size - helper to calculate size of each accounted object 126 * @size: size of area to allocate in bytes 127 * 128 * For each accounted object there is an extra space which is used to store 129 * obj_cgroup membership if kmemcg is not disabled. Charge it too. 130 */ 131 static inline size_t pcpu_obj_full_size(size_t size) 132 { 133 size_t extra_size = 0; 134 135 #ifdef CONFIG_MEMCG_KMEM 136 if (!mem_cgroup_kmem_disabled()) 137 extra_size += size / PCPU_MIN_ALLOC_SIZE * sizeof(struct obj_cgroup *); 138 #endif 139 140 return size * num_possible_cpus() + extra_size; 141 } 142 143 #ifdef CONFIG_PERCPU_STATS 144 145 #include <linux/spinlock.h> 146 147 struct percpu_stats { 148 u64 nr_alloc; /* lifetime # of allocations */ 149 u64 nr_dealloc; /* lifetime # of deallocations */ 150 u64 nr_cur_alloc; /* current # of allocations */ 151 u64 nr_max_alloc; /* max # of live allocations */ 152 u32 nr_chunks; /* current # of live chunks */ 153 u32 nr_max_chunks; /* max # of live chunks */ 154 size_t min_alloc_size; /* min allocation size */ 155 size_t max_alloc_size; /* max allocation size */ 156 }; 157 158 extern struct percpu_stats pcpu_stats; 159 extern struct pcpu_alloc_info pcpu_stats_ai; 160 161 /* 162 * For debug purposes. We don't care about the flexible array. 163 */ 164 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 165 { 166 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 167 168 /* initialize min_alloc_size to unit_size */ 169 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 170 } 171 172 /* 173 * pcpu_stats_area_alloc - increment area allocation stats 174 * @chunk: the location of the area being allocated 175 * @size: size of area to allocate in bytes 176 * 177 * CONTEXT: 178 * pcpu_lock. 179 */ 180 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 181 { 182 lockdep_assert_held(&pcpu_lock); 183 184 pcpu_stats.nr_alloc++; 185 pcpu_stats.nr_cur_alloc++; 186 pcpu_stats.nr_max_alloc = 187 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 188 pcpu_stats.min_alloc_size = 189 min(pcpu_stats.min_alloc_size, size); 190 pcpu_stats.max_alloc_size = 191 max(pcpu_stats.max_alloc_size, size); 192 193 chunk->nr_alloc++; 194 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 195 } 196 197 /* 198 * pcpu_stats_area_dealloc - decrement allocation stats 199 * @chunk: the location of the area being deallocated 200 * 201 * CONTEXT: 202 * pcpu_lock. 203 */ 204 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 205 { 206 lockdep_assert_held(&pcpu_lock); 207 208 pcpu_stats.nr_dealloc++; 209 pcpu_stats.nr_cur_alloc--; 210 211 chunk->nr_alloc--; 212 } 213 214 /* 215 * pcpu_stats_chunk_alloc - increment chunk stats 216 */ 217 static inline void pcpu_stats_chunk_alloc(void) 218 { 219 unsigned long flags; 220 spin_lock_irqsave(&pcpu_lock, flags); 221 222 pcpu_stats.nr_chunks++; 223 pcpu_stats.nr_max_chunks = 224 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 225 226 spin_unlock_irqrestore(&pcpu_lock, flags); 227 } 228 229 /* 230 * pcpu_stats_chunk_dealloc - decrement chunk stats 231 */ 232 static inline void pcpu_stats_chunk_dealloc(void) 233 { 234 unsigned long flags; 235 spin_lock_irqsave(&pcpu_lock, flags); 236 237 pcpu_stats.nr_chunks--; 238 239 spin_unlock_irqrestore(&pcpu_lock, flags); 240 } 241 242 #else 243 244 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 245 { 246 } 247 248 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 249 { 250 } 251 252 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 253 { 254 } 255 256 static inline void pcpu_stats_chunk_alloc(void) 257 { 258 } 259 260 static inline void pcpu_stats_chunk_dealloc(void) 261 { 262 } 263 264 #endif /* !CONFIG_PERCPU_STATS */ 265 266 #endif 267