Lines Matching refs:class
248 unsigned int class:CLASS_BITS + 1; member
486 *class_idx = zspage->class; in get_zspage_mapping()
492 return pool->size_class[zspage->class]; in zspage_class()
499 zspage->class = class_idx; in set_zspage_mapping()
521 static inline void class_stat_inc(struct size_class *class, in class_stat_inc() argument
524 class->stats.objs[type] += cnt; in class_stat_inc()
527 static inline void class_stat_dec(struct size_class *class, in class_stat_dec() argument
530 class->stats.objs[type] -= cnt; in class_stat_dec()
533 static inline unsigned long zs_stat_get(struct size_class *class, int type) in zs_stat_get() argument
535 return class->stats.objs[type]; in zs_stat_get()
555 static unsigned long zs_can_compact(struct size_class *class);
561 struct size_class *class; in zs_stats_size_show() local
576 class = pool->size_class[i]; in zs_stats_size_show()
578 if (class->index != i) in zs_stats_size_show()
583 seq_printf(s, " %5u %5u ", i, class->size); in zs_stats_size_show()
585 inuse_totals[fg] += zs_stat_get(class, fg); in zs_stats_size_show()
586 seq_printf(s, "%9lu ", zs_stat_get(class, fg)); in zs_stats_size_show()
589 obj_allocated = zs_stat_get(class, ZS_OBJS_ALLOCATED); in zs_stats_size_show()
590 obj_used = zs_stat_get(class, ZS_OBJS_INUSE); in zs_stats_size_show()
591 freeable = zs_can_compact(class); in zs_stats_size_show()
594 objs_per_zspage = class->objs_per_zspage; in zs_stats_size_show()
596 class->pages_per_zspage; in zs_stats_size_show()
600 class->pages_per_zspage, freeable); in zs_stats_size_show()
664 static int get_fullness_group(struct size_class *class, struct zspage *zspage) in get_fullness_group() argument
669 objs_per_zspage = class->objs_per_zspage; in get_fullness_group()
691 static void insert_zspage(struct size_class *class, in insert_zspage() argument
695 class_stat_inc(class, fullness, 1); in insert_zspage()
696 list_add(&zspage->list, &class->fullness_list[fullness]); in insert_zspage()
703 static void remove_zspage(struct size_class *class, in remove_zspage() argument
707 VM_BUG_ON(list_empty(&class->fullness_list[fullness])); in remove_zspage()
710 class_stat_dec(class, fullness, 1); in remove_zspage()
722 static int fix_fullness_group(struct size_class *class, struct zspage *zspage) in fix_fullness_group() argument
728 newfg = get_fullness_group(class, zspage); in fix_fullness_group()
732 remove_zspage(class, zspage, currfg); in fix_fullness_group()
733 insert_zspage(class, zspage, newfg); in fix_fullness_group()
848 static void __free_zspage(struct zs_pool *pool, struct size_class *class, in __free_zspage() argument
875 class_stat_dec(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); in __free_zspage()
876 atomic_long_sub(class->pages_per_zspage, &pool->pages_allocated); in __free_zspage()
879 static void free_zspage(struct zs_pool *pool, struct size_class *class, in free_zspage() argument
895 remove_zspage(class, zspage, ZS_INUSE_RATIO_0); in free_zspage()
896 __free_zspage(pool, class, zspage); in free_zspage()
900 static void init_zspage(struct size_class *class, struct zspage *zspage) in init_zspage() argument
916 while ((off += class->size) < PAGE_SIZE) { in init_zspage()
918 link += class->size / sizeof(*link); in init_zspage()
944 static void create_page_chain(struct size_class *class, struct zspage *zspage, in create_page_chain() argument
950 int nr_pages = class->pages_per_zspage; in create_page_chain()
967 if (unlikely(class->objs_per_zspage == 1 && in create_page_chain()
968 class->pages_per_zspage == 1)) in create_page_chain()
981 struct size_class *class, in alloc_zspage() argument
994 for (i = 0; i < class->pages_per_zspage; i++) { in alloc_zspage()
1011 create_page_chain(class, zspage, pages); in alloc_zspage()
1012 init_zspage(class, zspage); in alloc_zspage()
1018 static struct zspage *find_get_zspage(struct size_class *class) in find_get_zspage() argument
1024 zspage = list_first_entry_or_null(&class->fullness_list[i], in find_get_zspage()
1140 static bool zspage_full(struct size_class *class, struct zspage *zspage) in zspage_full() argument
1142 return get_zspage_inuse(zspage) == class->objs_per_zspage; in zspage_full()
1163 struct size_class *class; in zs_lookup_class_index() local
1165 class = pool->size_class[get_size_class_index(size)]; in zs_lookup_class_index()
1167 return class->index; in zs_lookup_class_index()
1200 struct size_class *class; in zs_map_object() local
1227 class = zspage_class(pool, zspage); in zs_map_object()
1228 off = offset_in_page(class->size * obj_idx); in zs_map_object()
1233 if (off + class->size <= PAGE_SIZE) { in zs_map_object()
1245 ret = __zs_map_object(area, pages, off, class->size); in zs_map_object()
1261 struct size_class *class; in zs_unmap_object() local
1267 class = zspage_class(pool, zspage); in zs_unmap_object()
1268 off = offset_in_page(class->size * obj_idx); in zs_unmap_object()
1271 if (off + class->size <= PAGE_SIZE) in zs_unmap_object()
1280 __zs_unmap_object(area, pages, off, class->size); in zs_unmap_object()
1313 struct size_class *class; in obj_malloc() local
1319 class = pool->size_class[zspage->class]; in obj_malloc()
1323 offset = obj * class->size; in obj_malloc()
1363 struct size_class *class; in zs_malloc() local
1376 class = pool->size_class[get_size_class_index(size)]; in zs_malloc()
1380 zspage = find_get_zspage(class); in zs_malloc()
1384 fix_fullness_group(class, zspage); in zs_malloc()
1386 class_stat_inc(class, ZS_OBJS_INUSE, 1); in zs_malloc()
1393 zspage = alloc_zspage(pool, class, gfp); in zs_malloc()
1401 newfg = get_fullness_group(class, zspage); in zs_malloc()
1402 insert_zspage(class, zspage, newfg); in zs_malloc()
1403 set_zspage_mapping(zspage, class->index, newfg); in zs_malloc()
1405 atomic_long_add(class->pages_per_zspage, &pool->pages_allocated); in zs_malloc()
1406 class_stat_inc(class, ZS_OBJS_ALLOCATED, class->objs_per_zspage); in zs_malloc()
1407 class_stat_inc(class, ZS_OBJS_INUSE, 1); in zs_malloc()
1450 struct size_class *class; in zs_free() local
1464 class = zspage_class(pool, zspage); in zs_free()
1466 class_stat_dec(class, ZS_OBJS_INUSE, 1); in zs_free()
1467 obj_free(class->size, obj); in zs_free()
1469 fullness = fix_fullness_group(class, zspage); in zs_free()
1471 free_zspage(pool, class, zspage); in zs_free()
1478 static void zs_object_copy(struct size_class *class, unsigned long dst, in zs_object_copy() argument
1488 s_size = d_size = class->size; in zs_object_copy()
1493 s_off = offset_in_page(class->size * s_objidx); in zs_object_copy()
1494 d_off = offset_in_page(class->size * d_objidx); in zs_object_copy()
1496 if (s_off + class->size > PAGE_SIZE) in zs_object_copy()
1499 if (d_off + class->size > PAGE_SIZE) in zs_object_copy()
1510 if (written == class->size) in zs_object_copy()
1531 s_size = class->size - written; in zs_object_copy()
1539 d_size = class->size - written; in zs_object_copy()
1552 static unsigned long find_alloced_obj(struct size_class *class, in find_alloced_obj() argument
1561 offset += class->size * index; in find_alloced_obj()
1567 offset += class->size; in find_alloced_obj()
1585 struct size_class *class = pool->size_class[src_zspage->class]; in migrate_zspage() local
1588 handle = find_alloced_obj(class, s_page, &obj_idx); in migrate_zspage()
1599 zs_object_copy(class, free_obj, used_obj); in migrate_zspage()
1602 obj_free(class->size, used_obj); in migrate_zspage()
1605 if (zspage_full(class, dst_zspage)) in migrate_zspage()
1614 static struct zspage *isolate_src_zspage(struct size_class *class) in isolate_src_zspage() argument
1620 zspage = list_first_entry_or_null(&class->fullness_list[fg], in isolate_src_zspage()
1623 remove_zspage(class, zspage, fg); in isolate_src_zspage()
1631 static struct zspage *isolate_dst_zspage(struct size_class *class) in isolate_dst_zspage() argument
1637 zspage = list_first_entry_or_null(&class->fullness_list[fg], in isolate_dst_zspage()
1640 remove_zspage(class, zspage, fg); in isolate_dst_zspage()
1655 static int putback_zspage(struct size_class *class, struct zspage *zspage) in putback_zspage() argument
1659 fullness = get_fullness_group(class, zspage); in putback_zspage()
1660 insert_zspage(class, zspage, fullness); in putback_zspage()
1661 set_zspage_mapping(zspage, class->index, fullness); in putback_zspage()
1755 static void replace_sub_page(struct size_class *class, struct zspage *zspage, in replace_sub_page() argument
1771 create_page_chain(class, zspage, pages); in replace_sub_page()
1802 struct size_class *class; in zs_page_migrate() local
1830 class = zspage_class(pool, zspage); in zs_page_migrate()
1846 addr += class->size) { in zs_page_migrate()
1858 replace_sub_page(class, zspage, newpage, page); in zs_page_migrate()
1906 struct size_class *class; in async_free_zspage() local
1915 class = pool->size_class[i]; in async_free_zspage()
1916 if (class->index != i) in async_free_zspage()
1920 list_splice_init(&class->fullness_list[ZS_INUSE_RATIO_0], in async_free_zspage()
1931 class = pool->size_class[class_idx]; in async_free_zspage()
1933 __free_zspage(pool, class, zspage); in async_free_zspage()
1972 static unsigned long zs_can_compact(struct size_class *class) in zs_can_compact() argument
1975 unsigned long obj_allocated = zs_stat_get(class, ZS_OBJS_ALLOCATED); in zs_can_compact()
1976 unsigned long obj_used = zs_stat_get(class, ZS_OBJS_INUSE); in zs_can_compact()
1982 obj_wasted /= class->objs_per_zspage; in zs_can_compact()
1984 return obj_wasted * class->pages_per_zspage; in zs_can_compact()
1988 struct size_class *class) in __zs_compact() argument
1999 while (zs_can_compact(class)) { in __zs_compact()
2003 dst_zspage = isolate_dst_zspage(class); in __zs_compact()
2009 src_zspage = isolate_src_zspage(class); in __zs_compact()
2016 fg = putback_zspage(class, src_zspage); in __zs_compact()
2020 free_zspage(pool, class, src_zspage); in __zs_compact()
2021 pages_freed += class->pages_per_zspage; in __zs_compact()
2025 if (get_fullness_group(class, dst_zspage) == ZS_INUSE_RATIO_100 in __zs_compact()
2027 putback_zspage(class, dst_zspage); in __zs_compact()
2038 putback_zspage(class, src_zspage); in __zs_compact()
2043 putback_zspage(class, dst_zspage); in __zs_compact()
2054 struct size_class *class; in zs_compact() local
2067 class = pool->size_class[i]; in zs_compact()
2068 if (class->index != i) in zs_compact()
2070 pages_freed += __zs_compact(pool, class); in zs_compact()
2106 struct size_class *class; in zs_shrinker_count() local
2112 class = pool->size_class[i]; in zs_shrinker_count()
2113 if (class->index != i) in zs_shrinker_count()
2116 pages_to_free += zs_can_compact(class); in zs_shrinker_count()
2198 struct size_class *class; in zs_create_pool() local
2244 class = kzalloc(sizeof(struct size_class), GFP_KERNEL); in zs_create_pool()
2245 if (!class) in zs_create_pool()
2248 class->size = size; in zs_create_pool()
2249 class->index = i; in zs_create_pool()
2250 class->pages_per_zspage = pages_per_zspage; in zs_create_pool()
2251 class->objs_per_zspage = objs_per_zspage; in zs_create_pool()
2252 pool->size_class[i] = class; in zs_create_pool()
2256 INIT_LIST_HEAD(&class->fullness_list[fullness]); in zs_create_pool()
2260 prev_class = class; in zs_create_pool()
2292 struct size_class *class = pool->size_class[i]; in zs_destroy_pool() local
2294 if (!class) in zs_destroy_pool()
2297 if (class->index != i) in zs_destroy_pool()
2301 if (list_empty(&class->fullness_list[fg])) in zs_destroy_pool()
2305 class->size, fg); in zs_destroy_pool()
2307 kfree(class); in zs_destroy_pool()