117 	struct bpf_local_storage *local_storage;  in __bpf_local_storage_free_trace_rcu()  local
122 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in __bpf_local_storage_free_trace_rcu()
124 		kfree(local_storage);  in __bpf_local_storage_free_trace_rcu()
126 		kfree_rcu(local_storage, rcu);  in __bpf_local_storage_free_trace_rcu()
131 	struct bpf_local_storage *local_storage;  in bpf_local_storage_free_rcu()  local
133 	local_storage = container_of(rcu, struct bpf_local_storage, rcu);  in bpf_local_storage_free_rcu()
134 	bpf_mem_cache_raw_free(local_storage);  in bpf_local_storage_free_rcu()
146 static void __bpf_local_storage_free(struct bpf_local_storage *local_storage,  in __bpf_local_storage_free()  argument
150 		kfree_rcu(local_storage, rcu);  in __bpf_local_storage_free()
152 		call_rcu_tasks_trace(&local_storage->rcu,  in __bpf_local_storage_free()
156 static void bpf_local_storage_free(struct bpf_local_storage *local_storage,  in bpf_local_storage_free()  argument
160 	if (!local_storage)  in bpf_local_storage_free()
164 		__bpf_local_storage_free(local_storage, reuse_now);  in bpf_local_storage_free()
169 		call_rcu_tasks_trace(&local_storage->rcu,  in bpf_local_storage_free()
176 		bpf_mem_cache_free(&smap->storage_ma, local_storage);  in bpf_local_storage_free()
183 		call_rcu(&local_storage->rcu, bpf_local_storage_free_rcu);  in bpf_local_storage_free()
253 static bool bpf_selem_unlink_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_unlink_storage_nolock()  argument
262 	owner = local_storage->owner;  in bpf_selem_unlink_storage_nolock()
272 						    &local_storage->list);  in bpf_selem_unlink_storage_nolock()
275 		local_storage->owner = NULL;  in bpf_selem_unlink_storage_nolock()
295 	if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) ==  in bpf_selem_unlink_storage_nolock()
297 		RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL);  in bpf_selem_unlink_storage_nolock()
301 	if (rcu_access_pointer(local_storage->smap) == smap)  in bpf_selem_unlink_storage_nolock()
302 		RCU_INIT_POINTER(local_storage->smap, NULL);  in bpf_selem_unlink_storage_nolock()
307 static bool check_storage_bpf_ma(struct bpf_local_storage *local_storage,  in check_storage_bpf_ma()  argument
330 		n = rcu_dereference_check(hlist_first_rcu(&local_storage->list),  in check_storage_bpf_ma()
346 	struct bpf_local_storage *local_storage;  in bpf_selem_unlink_storage()  local
354 	local_storage = rcu_dereference_check(selem->local_storage,  in bpf_selem_unlink_storage()
356 	storage_smap = rcu_dereference_check(local_storage->smap,  in bpf_selem_unlink_storage()
358 	bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, selem);  in bpf_selem_unlink_storage()
360 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_selem_unlink_storage()
363 			local_storage, selem, true, reuse_now);  in bpf_selem_unlink_storage()
364 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_selem_unlink_storage()
367 		bpf_local_storage_free(local_storage, storage_smap, bpf_ma, reuse_now);  in bpf_selem_unlink_storage()
370 void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,  in bpf_selem_link_storage_nolock()  argument
373 	RCU_INIT_POINTER(selem->local_storage, local_storage);  in bpf_selem_link_storage_nolock()
374 	hlist_add_head_rcu(&selem->snode, &local_storage->list);  in bpf_selem_link_storage_nolock()
419 bpf_local_storage_lookup(struct bpf_local_storage *local_storage,  in bpf_local_storage_lookup()  argument
427 	sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
433 	hlist_for_each_entry_rcu(selem, &local_storage->list, snode,  in bpf_local_storage_lookup()
450 		raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_lookup()
452 			rcu_assign_pointer(local_storage->cache[smap->cache_idx],  in bpf_local_storage_lookup()
454 		raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_lookup()
557 	struct bpf_local_storage *local_storage;  in bpf_local_storage_update()  local
571 	local_storage = rcu_dereference_check(*owner_storage(smap, owner),  in bpf_local_storage_update()
573 	if (!local_storage || hlist_empty(&local_storage->list)) {  in bpf_local_storage_update()
599 			bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
617 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_update()
620 	if (unlikely(hlist_empty(&local_storage->list))) {  in bpf_local_storage_update()
630 	old_sdata = bpf_local_storage_lookup(local_storage, smap, false);  in bpf_local_storage_update()
647 	bpf_selem_link_storage_nolock(local_storage, selem);  in bpf_local_storage_update()
652 		bpf_selem_unlink_storage_nolock(local_storage, SELEM(old_sdata),  in bpf_local_storage_update()
657 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_update()
730 void bpf_local_storage_destroy(struct bpf_local_storage *local_storage)  in bpf_local_storage_destroy()  argument
738 	storage_smap = rcu_dereference_check(local_storage->smap, bpf_rcu_lock_held());  in bpf_local_storage_destroy()
739 	bpf_ma = check_storage_bpf_ma(local_storage, storage_smap, NULL);  in bpf_local_storage_destroy()
750 	raw_spin_lock_irqsave(&local_storage->lock, flags);  in bpf_local_storage_destroy()
751 	hlist_for_each_entry_safe(selem, n, &local_storage->list, snode) {  in bpf_local_storage_destroy()
763 			local_storage, selem, true, true);  in bpf_local_storage_destroy()
765 	raw_spin_unlock_irqrestore(&local_storage->lock, flags);  in bpf_local_storage_destroy()
768 		bpf_local_storage_free(local_storage, storage_smap, bpf_ma, true);  in bpf_local_storage_destroy()