18fa3ed80SDennis Zhou #ifndef _MM_PERCPU_INTERNAL_H 28fa3ed80SDennis Zhou #define _MM_PERCPU_INTERNAL_H 38fa3ed80SDennis Zhou 48fa3ed80SDennis Zhou #include <linux/types.h> 58fa3ed80SDennis Zhou #include <linux/percpu.h> 68fa3ed80SDennis Zhou 78fa3ed80SDennis Zhou struct pcpu_chunk { 830a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 930a5b536SDennis Zhou int nr_alloc; /* # of allocations */ 1030a5b536SDennis Zhou size_t max_alloc_size; /* largest allocation size */ 1130a5b536SDennis Zhou #endif 1230a5b536SDennis Zhou 138fa3ed80SDennis Zhou struct list_head list; /* linked to pcpu_slot lists */ 148fa3ed80SDennis Zhou int free_size; /* free bytes in the chunk */ 158fa3ed80SDennis Zhou int contig_hint; /* max contiguous size hint */ 168fa3ed80SDennis Zhou void *base_addr; /* base address of this chunk */ 178fa3ed80SDennis Zhou 188fa3ed80SDennis Zhou int map_used; /* # of map entries used before the sentry */ 198fa3ed80SDennis Zhou int map_alloc; /* # of map entries allocated */ 208fa3ed80SDennis Zhou int *map; /* allocation map */ 218fa3ed80SDennis Zhou struct list_head map_extend_list;/* on pcpu_map_extend_chunks */ 228fa3ed80SDennis Zhou 238fa3ed80SDennis Zhou void *data; /* chunk data */ 248fa3ed80SDennis Zhou int first_free; /* no free below this */ 258fa3ed80SDennis Zhou bool immutable; /* no [de]population allowed */ 26e2266705SDennis Zhou (Facebook) int start_offset; /* the overlap with the previous 27e2266705SDennis Zhou (Facebook) region to have a page aligned 28e2266705SDennis Zhou (Facebook) base_addr */ 296b9d7c8eSDennis Zhou (Facebook) int end_offset; /* additional area required to 306b9d7c8eSDennis Zhou (Facebook) have the region end page 316b9d7c8eSDennis Zhou (Facebook) aligned */ 32c0ebfdc3SDennis Zhou (Facebook) 33c0ebfdc3SDennis Zhou (Facebook) int nr_pages; /* # of pages served by this chunk */ 348fa3ed80SDennis Zhou int nr_populated; /* # of populated pages */ 358fa3ed80SDennis Zhou unsigned long populated[]; /* populated bitmap */ 368fa3ed80SDennis Zhou }; 378fa3ed80SDennis Zhou 388fa3ed80SDennis Zhou extern spinlock_t pcpu_lock; 398fa3ed80SDennis Zhou 408fa3ed80SDennis Zhou extern struct list_head *pcpu_slot; 418fa3ed80SDennis Zhou extern int pcpu_nr_slots; 426b9b6f39SDennis Zhou (Facebook) extern int pcpu_nr_empty_pop_pages; 438fa3ed80SDennis Zhou 448fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_first_chunk; 458fa3ed80SDennis Zhou extern struct pcpu_chunk *pcpu_reserved_chunk; 468fa3ed80SDennis Zhou 4730a5b536SDennis Zhou #ifdef CONFIG_PERCPU_STATS 4830a5b536SDennis Zhou 4930a5b536SDennis Zhou #include <linux/spinlock.h> 5030a5b536SDennis Zhou 5130a5b536SDennis Zhou struct percpu_stats { 5230a5b536SDennis Zhou u64 nr_alloc; /* lifetime # of allocations */ 5330a5b536SDennis Zhou u64 nr_dealloc; /* lifetime # of deallocations */ 5430a5b536SDennis Zhou u64 nr_cur_alloc; /* current # of allocations */ 5530a5b536SDennis Zhou u64 nr_max_alloc; /* max # of live allocations */ 5630a5b536SDennis Zhou u32 nr_chunks; /* current # of live chunks */ 5730a5b536SDennis Zhou u32 nr_max_chunks; /* max # of live chunks */ 5830a5b536SDennis Zhou size_t min_alloc_size; /* min allocaiton size */ 5930a5b536SDennis Zhou size_t max_alloc_size; /* max allocation size */ 6030a5b536SDennis Zhou }; 6130a5b536SDennis Zhou 6230a5b536SDennis Zhou extern struct percpu_stats pcpu_stats; 6330a5b536SDennis Zhou extern struct pcpu_alloc_info pcpu_stats_ai; 6430a5b536SDennis Zhou 6530a5b536SDennis Zhou /* 6630a5b536SDennis Zhou * For debug purposes. We don't care about the flexible array. 6730a5b536SDennis Zhou */ 6830a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 6930a5b536SDennis Zhou { 7030a5b536SDennis Zhou memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info)); 7130a5b536SDennis Zhou 7230a5b536SDennis Zhou /* initialize min_alloc_size to unit_size */ 7330a5b536SDennis Zhou pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size; 7430a5b536SDennis Zhou } 7530a5b536SDennis Zhou 7630a5b536SDennis Zhou /* 7730a5b536SDennis Zhou * pcpu_stats_area_alloc - increment area allocation stats 7830a5b536SDennis Zhou * @chunk: the location of the area being allocated 7930a5b536SDennis Zhou * @size: size of area to allocate in bytes 8030a5b536SDennis Zhou * 8130a5b536SDennis Zhou * CONTEXT: 8230a5b536SDennis Zhou * pcpu_lock. 8330a5b536SDennis Zhou */ 8430a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 8530a5b536SDennis Zhou { 8630a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 8730a5b536SDennis Zhou 8830a5b536SDennis Zhou pcpu_stats.nr_alloc++; 8930a5b536SDennis Zhou pcpu_stats.nr_cur_alloc++; 9030a5b536SDennis Zhou pcpu_stats.nr_max_alloc = 9130a5b536SDennis Zhou max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc); 9230a5b536SDennis Zhou pcpu_stats.min_alloc_size = 9330a5b536SDennis Zhou min(pcpu_stats.min_alloc_size, size); 9430a5b536SDennis Zhou pcpu_stats.max_alloc_size = 9530a5b536SDennis Zhou max(pcpu_stats.max_alloc_size, size); 9630a5b536SDennis Zhou 9730a5b536SDennis Zhou chunk->nr_alloc++; 9830a5b536SDennis Zhou chunk->max_alloc_size = max(chunk->max_alloc_size, size); 9930a5b536SDennis Zhou } 10030a5b536SDennis Zhou 10130a5b536SDennis Zhou /* 10230a5b536SDennis Zhou * pcpu_stats_area_dealloc - decrement allocation stats 10330a5b536SDennis Zhou * @chunk: the location of the area being deallocated 10430a5b536SDennis Zhou * 10530a5b536SDennis Zhou * CONTEXT: 10630a5b536SDennis Zhou * pcpu_lock. 10730a5b536SDennis Zhou */ 10830a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 10930a5b536SDennis Zhou { 11030a5b536SDennis Zhou lockdep_assert_held(&pcpu_lock); 11130a5b536SDennis Zhou 11230a5b536SDennis Zhou pcpu_stats.nr_dealloc++; 11330a5b536SDennis Zhou pcpu_stats.nr_cur_alloc--; 11430a5b536SDennis Zhou 11530a5b536SDennis Zhou chunk->nr_alloc--; 11630a5b536SDennis Zhou } 11730a5b536SDennis Zhou 11830a5b536SDennis Zhou /* 11930a5b536SDennis Zhou * pcpu_stats_chunk_alloc - increment chunk stats 12030a5b536SDennis Zhou */ 12130a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 12230a5b536SDennis Zhou { 123303abfdfSDennis Zhou unsigned long flags; 124303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 12530a5b536SDennis Zhou 12630a5b536SDennis Zhou pcpu_stats.nr_chunks++; 12730a5b536SDennis Zhou pcpu_stats.nr_max_chunks = 12830a5b536SDennis Zhou max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks); 12930a5b536SDennis Zhou 130303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 13130a5b536SDennis Zhou } 13230a5b536SDennis Zhou 13330a5b536SDennis Zhou /* 13430a5b536SDennis Zhou * pcpu_stats_chunk_dealloc - decrement chunk stats 13530a5b536SDennis Zhou */ 13630a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 13730a5b536SDennis Zhou { 138303abfdfSDennis Zhou unsigned long flags; 139303abfdfSDennis Zhou spin_lock_irqsave(&pcpu_lock, flags); 14030a5b536SDennis Zhou 14130a5b536SDennis Zhou pcpu_stats.nr_chunks--; 14230a5b536SDennis Zhou 143303abfdfSDennis Zhou spin_unlock_irqrestore(&pcpu_lock, flags); 14430a5b536SDennis Zhou } 14530a5b536SDennis Zhou 14630a5b536SDennis Zhou #else 14730a5b536SDennis Zhou 14830a5b536SDennis Zhou static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai) 14930a5b536SDennis Zhou { 15030a5b536SDennis Zhou } 15130a5b536SDennis Zhou 15230a5b536SDennis Zhou static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size) 15330a5b536SDennis Zhou { 15430a5b536SDennis Zhou } 15530a5b536SDennis Zhou 15630a5b536SDennis Zhou static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk) 15730a5b536SDennis Zhou { 15830a5b536SDennis Zhou } 15930a5b536SDennis Zhou 16030a5b536SDennis Zhou static inline void pcpu_stats_chunk_alloc(void) 16130a5b536SDennis Zhou { 16230a5b536SDennis Zhou } 16330a5b536SDennis Zhou 16430a5b536SDennis Zhou static inline void pcpu_stats_chunk_dealloc(void) 16530a5b536SDennis Zhou { 16630a5b536SDennis Zhou } 16730a5b536SDennis Zhou 16830a5b536SDennis Zhou #endif /* !CONFIG_PERCPU_STATS */ 16930a5b536SDennis Zhou 1708fa3ed80SDennis Zhou #endif 171