Lines Matching refs:stock
2207 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock);
2208 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2213 static inline struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
2217 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
2240 struct memcg_stock_pcp *stock; in consume_stock() local
2249 stock = this_cpu_ptr(&memcg_stock); in consume_stock()
2250 if (memcg == READ_ONCE(stock->cached) && stock->nr_pages >= nr_pages) { in consume_stock()
2251 stock->nr_pages -= nr_pages; in consume_stock()
2263 static void drain_stock(struct memcg_stock_pcp *stock) in drain_stock() argument
2265 struct mem_cgroup *old = READ_ONCE(stock->cached); in drain_stock()
2270 if (stock->nr_pages) { in drain_stock()
2271 page_counter_uncharge(&old->memory, stock->nr_pages); in drain_stock()
2273 page_counter_uncharge(&old->memsw, stock->nr_pages); in drain_stock()
2274 stock->nr_pages = 0; in drain_stock()
2278 WRITE_ONCE(stock->cached, NULL); in drain_stock()
2283 struct memcg_stock_pcp *stock; in drain_local_stock() local
2294 stock = this_cpu_ptr(&memcg_stock); in drain_local_stock()
2295 old = drain_obj_stock(stock); in drain_local_stock()
2296 drain_stock(stock); in drain_local_stock()
2297 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags); in drain_local_stock()
2310 struct memcg_stock_pcp *stock; in __refill_stock() local
2312 stock = this_cpu_ptr(&memcg_stock); in __refill_stock()
2313 if (READ_ONCE(stock->cached) != memcg) { /* reset if necessary */ in __refill_stock()
2314 drain_stock(stock); in __refill_stock()
2316 WRITE_ONCE(stock->cached, memcg); in __refill_stock()
2318 stock->nr_pages += nr_pages; in __refill_stock()
2320 if (stock->nr_pages > MEMCG_CHARGE_BATCH) in __refill_stock()
2321 drain_stock(stock); in __refill_stock()
2353 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); in drain_all_stock() local
2358 memcg = READ_ONCE(stock->cached); in drain_all_stock()
2359 if (memcg && stock->nr_pages && in drain_all_stock()
2362 else if (obj_stock_flush_required(stock, root_memcg)) in drain_all_stock()
2367 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) { in drain_all_stock()
2369 drain_local_stock(&stock->work); in drain_all_stock()
2371 schedule_work_on(cpu, &stock->work); in drain_all_stock()
2380 struct memcg_stock_pcp *stock; in memcg_hotplug_cpu_dead() local
2382 stock = &per_cpu(memcg_stock, cpu); in memcg_hotplug_cpu_dead()
2383 drain_stock(stock); in memcg_hotplug_cpu_dead()
3171 struct memcg_stock_pcp *stock; in mod_objcg_state() local
3177 stock = this_cpu_ptr(&memcg_stock); in mod_objcg_state()
3184 if (READ_ONCE(stock->cached_objcg) != objcg) { in mod_objcg_state()
3185 old = drain_obj_stock(stock); in mod_objcg_state()
3187 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in mod_objcg_state()
3189 WRITE_ONCE(stock->cached_objcg, objcg); in mod_objcg_state()
3190 stock->cached_pgdat = pgdat; in mod_objcg_state()
3191 } else if (stock->cached_pgdat != pgdat) { in mod_objcg_state()
3193 struct pglist_data *oldpg = stock->cached_pgdat; in mod_objcg_state()
3195 if (stock->nr_slab_reclaimable_b) { in mod_objcg_state()
3197 stock->nr_slab_reclaimable_b); in mod_objcg_state()
3198 stock->nr_slab_reclaimable_b = 0; in mod_objcg_state()
3200 if (stock->nr_slab_unreclaimable_b) { in mod_objcg_state()
3202 stock->nr_slab_unreclaimable_b); in mod_objcg_state()
3203 stock->nr_slab_unreclaimable_b = 0; in mod_objcg_state()
3205 stock->cached_pgdat = pgdat; in mod_objcg_state()
3208 bytes = (idx == NR_SLAB_RECLAIMABLE_B) ? &stock->nr_slab_reclaimable_b in mod_objcg_state()
3209 : &stock->nr_slab_unreclaimable_b; in mod_objcg_state()
3236 struct memcg_stock_pcp *stock; in consume_obj_stock() local
3242 stock = this_cpu_ptr(&memcg_stock); in consume_obj_stock()
3243 if (objcg == READ_ONCE(stock->cached_objcg) && stock->nr_bytes >= nr_bytes) { in consume_obj_stock()
3244 stock->nr_bytes -= nr_bytes; in consume_obj_stock()
3253 static struct obj_cgroup *drain_obj_stock(struct memcg_stock_pcp *stock) in drain_obj_stock() argument
3255 struct obj_cgroup *old = READ_ONCE(stock->cached_objcg); in drain_obj_stock()
3260 if (stock->nr_bytes) { in drain_obj_stock()
3261 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT; in drain_obj_stock()
3262 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1); in drain_obj_stock()
3286 stock->nr_bytes = 0; in drain_obj_stock()
3292 if (stock->nr_slab_reclaimable_b || stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3293 if (stock->nr_slab_reclaimable_b) { in drain_obj_stock()
3294 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3296 stock->nr_slab_reclaimable_b); in drain_obj_stock()
3297 stock->nr_slab_reclaimable_b = 0; in drain_obj_stock()
3299 if (stock->nr_slab_unreclaimable_b) { in drain_obj_stock()
3300 mod_objcg_mlstate(old, stock->cached_pgdat, in drain_obj_stock()
3302 stock->nr_slab_unreclaimable_b); in drain_obj_stock()
3303 stock->nr_slab_unreclaimable_b = 0; in drain_obj_stock()
3305 stock->cached_pgdat = NULL; in drain_obj_stock()
3308 WRITE_ONCE(stock->cached_objcg, NULL); in drain_obj_stock()
3316 static bool obj_stock_flush_required(struct memcg_stock_pcp *stock, in obj_stock_flush_required() argument
3319 struct obj_cgroup *objcg = READ_ONCE(stock->cached_objcg); in obj_stock_flush_required()
3334 struct memcg_stock_pcp *stock; in refill_obj_stock() local
3341 stock = this_cpu_ptr(&memcg_stock); in refill_obj_stock()
3342 if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */ in refill_obj_stock()
3343 old = drain_obj_stock(stock); in refill_obj_stock()
3345 WRITE_ONCE(stock->cached_objcg, objcg); in refill_obj_stock()
3346 stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes) in refill_obj_stock()
3350 stock->nr_bytes += nr_bytes; in refill_obj_stock()
3352 if (allow_uncharge && (stock->nr_bytes > PAGE_SIZE)) { in refill_obj_stock()
3353 nr_pages = stock->nr_bytes >> PAGE_SHIFT; in refill_obj_stock()
3354 stock->nr_bytes &= (PAGE_SIZE - 1); in refill_obj_stock()