Lines Matching refs:fbatch

53 	struct folio_batch fbatch;  member
195 static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn) in folio_batch_move_lru() argument
201 for (i = 0; i < folio_batch_count(fbatch); i++) { in folio_batch_move_lru()
202 struct folio *folio = fbatch->folios[i]; in folio_batch_move_lru()
216 folios_put(fbatch->folios, folio_batch_count(fbatch)); in folio_batch_move_lru()
217 folio_batch_reinit(fbatch); in folio_batch_move_lru()
220 static void folio_batch_add_and_move(struct folio_batch *fbatch, in folio_batch_add_and_move() argument
223 if (folio_batch_add(fbatch, folio) && !folio_test_large(folio) && in folio_batch_add_and_move()
226 folio_batch_move_lru(fbatch, move_fn); in folio_batch_add_and_move()
250 struct folio_batch *fbatch; in folio_rotate_reclaimable() local
255 fbatch = this_cpu_ptr(&lru_rotate.fbatch); in folio_rotate_reclaimable()
256 folio_batch_add_and_move(fbatch, folio, lru_move_tail_fn); in folio_rotate_reclaimable()
338 struct folio_batch *fbatch = &per_cpu(cpu_fbatches.activate, cpu); in folio_activate_drain() local
340 if (folio_batch_count(fbatch)) in folio_activate_drain()
341 folio_batch_move_lru(fbatch, folio_activate_fn); in folio_activate_drain()
348 struct folio_batch *fbatch; in folio_activate() local
352 fbatch = this_cpu_ptr(&cpu_fbatches.activate); in folio_activate()
353 folio_batch_add_and_move(fbatch, folio, folio_activate_fn); in folio_activate()
378 struct folio_batch *fbatch; in __lru_cache_activate_folio() local
382 fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); in __lru_cache_activate_folio()
394 for (i = folio_batch_count(fbatch) - 1; i >= 0; i--) { in __lru_cache_activate_folio()
395 struct folio *batch_folio = fbatch->folios[i]; in __lru_cache_activate_folio()
495 struct folio_batch *fbatch; in folio_add_lru() local
508 fbatch = this_cpu_ptr(&cpu_fbatches.lru_add); in folio_add_lru()
509 folio_batch_add_and_move(fbatch, folio, lru_add_fn); in folio_add_lru()
641 struct folio_batch *fbatch = &fbatches->lru_add; in lru_add_drain_cpu() local
643 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
644 folio_batch_move_lru(fbatch, lru_add_fn); in lru_add_drain_cpu()
646 fbatch = &per_cpu(lru_rotate.fbatch, cpu); in lru_add_drain_cpu()
648 if (data_race(folio_batch_count(fbatch))) { in lru_add_drain_cpu()
653 folio_batch_move_lru(fbatch, lru_move_tail_fn); in lru_add_drain_cpu()
657 fbatch = &fbatches->lru_deactivate_file; in lru_add_drain_cpu()
658 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
659 folio_batch_move_lru(fbatch, lru_deactivate_file_fn); in lru_add_drain_cpu()
661 fbatch = &fbatches->lru_deactivate; in lru_add_drain_cpu()
662 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
663 folio_batch_move_lru(fbatch, lru_deactivate_fn); in lru_add_drain_cpu()
665 fbatch = &fbatches->lru_lazyfree; in lru_add_drain_cpu()
666 if (folio_batch_count(fbatch)) in lru_add_drain_cpu()
667 folio_batch_move_lru(fbatch, lru_lazyfree_fn); in lru_add_drain_cpu()
684 struct folio_batch *fbatch; in deactivate_file_folio() local
692 fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate_file); in deactivate_file_folio()
693 folio_batch_add_and_move(fbatch, folio, lru_deactivate_file_fn); in deactivate_file_folio()
709 struct folio_batch *fbatch; in folio_deactivate() local
713 fbatch = this_cpu_ptr(&cpu_fbatches.lru_deactivate); in folio_deactivate()
714 folio_batch_add_and_move(fbatch, folio, lru_deactivate_fn); in folio_deactivate()
731 struct folio_batch *fbatch; in folio_mark_lazyfree() local
735 fbatch = this_cpu_ptr(&cpu_fbatches.lru_lazyfree); in folio_mark_lazyfree()
736 folio_batch_add_and_move(fbatch, folio, lru_lazyfree_fn); in folio_mark_lazyfree()
788 data_race(folio_batch_count(&per_cpu(lru_rotate.fbatch, cpu))) || in cpu_needs_drain()
1036 void __folio_batch_release(struct folio_batch *fbatch) in __folio_batch_release() argument
1038 if (!fbatch->percpu_pvec_drained) { in __folio_batch_release()
1040 fbatch->percpu_pvec_drained = true; in __folio_batch_release()
1042 release_pages(fbatch->folios, folio_batch_count(fbatch)); in __folio_batch_release()
1043 folio_batch_reinit(fbatch); in __folio_batch_release()
1056 void folio_batch_remove_exceptionals(struct folio_batch *fbatch) in folio_batch_remove_exceptionals() argument
1060 for (i = 0, j = 0; i < folio_batch_count(fbatch); i++) { in folio_batch_remove_exceptionals()
1061 struct folio *folio = fbatch->folios[i]; in folio_batch_remove_exceptionals()
1063 fbatch->folios[j++] = folio; in folio_batch_remove_exceptionals()
1065 fbatch->nr = j; in folio_batch_remove_exceptionals()