1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _MM_PERCPU_INTERNAL_H 3 #define _MM_PERCPU_INTERNAL_H 4 5 #include <linux/types.h> 6 #include <linux/percpu.h> 7 8 /* 9 * pcpu_block_md is the metadata block struct. 10 * Each chunk's bitmap is split into a number of full blocks. 11 * All units are in terms of bits. 12 */ 13 struct pcpu_block_md { 14 int contig_hint; /* contig hint for block */ 15 int contig_hint_start; /* block relative starting 16 position of the contig hint */ 17 int left_free; /* size of free space along 18 the left side of the block */ 19 int right_free; /* size of free space along 20 the right side of the block */ 21 int first_free; /* block position of first free */ 22 }; 23 24 struct pcpu_chunk { 25 #ifdef CONFIG_PERCPU_STATS 26 int nr_alloc; /* # of allocations */ 27 size_t max_alloc_size; /* largest allocation size */ 28 #endif 29 30 struct list_head list; /* linked to pcpu_slot lists */ 31 int free_bytes; /* free bytes in the chunk */ 32 int contig_bits; /* max contiguous size hint */ 33 int contig_bits_start; /* contig_bits starting 34 offset */ 35 void *base_addr; /* base address of this chunk */ 36 37 unsigned long *alloc_map; /* allocation map */ 38 unsigned long *bound_map; /* boundary map */ 39 struct pcpu_block_md *md_blocks; /* metadata blocks */ 40 41 void *data; /* chunk data */ 42 int first_bit; /* no free below this */ 43 bool immutable; /* no [de]population allowed */ 44 int start_offset; /* the overlap with the previous 45 region to have a page aligned 46 base_addr */ 47 int end_offset; /* additional area required to 48 have the region end page 49 aligned */ 50 51 int nr_pages; /* # of pages served by this chunk */ 52 int nr_populated; /* # of populated pages */ 53 int nr_empty_pop_pages; /* # of empty populated pages */ 54 unsigned long populated[]; /* populated bitmap */ 55 }; 56 57 extern spinlock_t pcpu_lock; 58 59 extern struct list_head *pcpu_slot; 60 extern int pcpu_nr_slots; 61 extern int pcpu_nr_empty_pop_pages; 62 63 extern struct pcpu_chunk *pcpu_first_chunk; 64 extern struct pcpu_chunk *pcpu_reserved_chunk; 65 66 /** 67 * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks 68 * @chunk: chunk of interest 69 * 70 * This conversion is from the number of physical pages that the chunk 71 * serves to the number of bitmap blocks used. 72 */ 73 static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk) 74 { 75 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE; 76 } 77 78 /** 79 * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap 80 * @pages: number of physical pages 81 * 82 * This conversion is from physical pages to the number of bits 83 * required in the bitmap. 84 */ 85 static inline int pcpu_nr_pages_to_map_bits(int pages) 86 { 87 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE; 88 } 89 90 /** 91 * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap 92 * @chunk: chunk of interest 93 * 94 * This conversion is from the number of physical pages that the chunk 95 * serves to the number of bits in the bitmap. 96 */ 97 static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk) 98 { 99 return pcpu_nr_pages_to_map_bits(chunk->nr_pages); 100 } 101 102 #ifdef CONFIG_PERCPU_STATS 103 104 #include <linux/spinlock.h> 105 106 struct percpu_stats { 107 u64 nr_alloc; /* lifetime # of allocations */ 108 u64 nr_dealloc; /* lifetime # of deallocations */ 109 u64 nr_cur_alloc; /* current # of allocations */ 110 u64 nr_max_alloc; /* max # of live allocations */ 111 u32 nr_chunks; /* current # of live chunks */ 112 u32 nr_max_chunks; /* max # of live chunks */ 113 size_t min_alloc_size; /* min allocaiton size */ 114 size_t max_alloc_size; /* max allocation size */ 115 }; 116 117 extern struct percpu_stats pcpu_stats; 118 extern struct pcpu_alloc_info pcpu_stats_ai; 119 120 /* 121 * For debug purposes. We don't care about the flexible array. 122 */ 123 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 124 { 125 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 126 127 /* initialize min_alloc_size to unit_size */ 128 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 129 } 130 131 /* 132 * pcpu_stats_area_alloc - increment area allocation stats 133 * @chunk: the location of the area being allocated 134 * @size: size of area to allocate in bytes 135 * 136 * CONTEXT: 137 * pcpu_lock. 138 */ 139 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 140 { 141 lockdep_assert_held(&pcpu_lock); 142 143 pcpu_stats.nr_alloc++; 144 pcpu_stats.nr_cur_alloc++; 145 pcpu_stats.nr_max_alloc = 146 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 147 pcpu_stats.min_alloc_size = 148 min(pcpu_stats.min_alloc_size, size); 149 pcpu_stats.max_alloc_size = 150 max(pcpu_stats.max_alloc_size, size); 151 152 chunk->nr_alloc++; 153 chunk->max_alloc_size = max(chunk->max_alloc_size, size); 154 } 155 156 /* 157 * pcpu_stats_area_dealloc - decrement allocation stats 158 * @chunk: the location of the area being deallocated 159 * 160 * CONTEXT: 161 * pcpu_lock. 162 */ 163 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 164 { 165 lockdep_assert_held(&pcpu_lock); 166 167 pcpu_stats.nr_dealloc++; 168 pcpu_stats.nr_cur_alloc--; 169 170 chunk->nr_alloc--; 171 } 172 173 /* 174 * pcpu_stats_chunk_alloc - increment chunk stats 175 */ 176 static inline void pcpu_stats_chunk_alloc(void) 177 { 178 unsigned long flags; 179 spin_lock_irqsave(&pcpu_lock, flags); 180 181 pcpu_stats.nr_chunks++; 182 pcpu_stats.nr_max_chunks = 183 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 184 185 spin_unlock_irqrestore(&pcpu_lock, flags); 186 } 187 188 /* 189 * pcpu_stats_chunk_dealloc - decrement chunk stats 190 */ 191 static inline void pcpu_stats_chunk_dealloc(void) 192 { 193 unsigned long flags; 194 spin_lock_irqsave(&pcpu_lock, flags); 195 196 pcpu_stats.nr_chunks--; 197 198 spin_unlock_irqrestore(&pcpu_lock, flags); 199 } 200 201 #else 202 203 static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 204 { 205 } 206 207 static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 208 { 209 } 210 211 static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 212 { 213 } 214 215 static inline void pcpu_stats_chunk_alloc(void) 216 { 217 } 218 219 static inline void pcpu_stats_chunk_dealloc(void) 220 { 221 } 222 223 #endif /* !CONFIG_PERCPU_STATS */ 224 225 #endif 226