107 bool bpf_prog_has_trampoline(const struct bpf_prog *prog)  in bpf_prog_has_trampoline()  argument
109 	enum bpf_attach_type eatype = prog->expected_attach_type;  in bpf_prog_has_trampoline()
110 	enum bpf_prog_type ptype = prog->type;  in bpf_prog_has_trampoline()
247 			*ip_arg |= link->link.prog->call_get_func_ip;  in bpf_trampoline_get_progs()
491 static enum bpf_tramp_prog_type bpf_attach_type_to_tramp(struct bpf_prog *prog)  in bpf_attach_type_to_tramp()  argument
493 	switch (prog->expected_attach_type) {  in bpf_attach_type_to_tramp()
501 		if (!prog->aux->attach_func_proto->type)  in bpf_attach_type_to_tramp()
520 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_link_prog()
534 		tr->extension_prog = link->link.prog;  in __bpf_trampoline_link_prog()
536 					  link->link.prog->bpf_func);  in __bpf_trampoline_link_prog()
544 		if (link_exiting->link.prog != link->link.prog)  in __bpf_trampoline_link_prog()
575 	kind = bpf_attach_type_to_tramp(link->link.prog);  in __bpf_trampoline_unlink_prog()
626 static struct bpf_shim_tramp_link *cgroup_shim_alloc(const struct bpf_prog *prog,  in cgroup_shim_alloc()  argument
647 	p->aux->attach_func_proto = prog->aux->attach_func_proto;  in cgroup_shim_alloc()
648 	p->aux->attach_btf_id = prog->aux->attach_btf_id;  in cgroup_shim_alloc()
649 	p->aux->attach_btf = prog->aux->attach_btf;  in cgroup_shim_alloc()
669 			struct bpf_prog *p = link->link.prog;  in cgroup_shim_find()
679 int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog,  in bpf_trampoline_link_cgroup_shim()  argument
689 	err = bpf_check_attach_target(NULL, prog, NULL,  in bpf_trampoline_link_cgroup_shim()
690 				      prog->aux->attach_btf_id,  in bpf_trampoline_link_cgroup_shim()
695 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_link_cgroup_shim()
696 					 prog->aux->attach_btf_id);  in bpf_trampoline_link_cgroup_shim()
698 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_link_cgroup_shim()
717 	shim_link = cgroup_shim_alloc(prog, bpf_func, cgroup_atype);  in bpf_trampoline_link_cgroup_shim()
745 void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog)  in bpf_trampoline_unlink_cgroup_shim()  argument
752 	key = bpf_trampoline_compute_key(NULL, prog->aux->attach_btf,  in bpf_trampoline_unlink_cgroup_shim()
753 					 prog->aux->attach_btf_id);  in bpf_trampoline_unlink_cgroup_shim()
755 	bpf_lsm_find_cgroup_shim(prog, &bpf_func);  in bpf_trampoline_unlink_cgroup_shim()
848 static u64 notrace __bpf_prog_enter_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)  in __bpf_prog_enter_recur()  argument
856 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_recur()
857 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_recur()
863 static void notrace update_prog_stats(struct bpf_prog *prog,  in update_prog_stats()  argument
877 		stats = this_cpu_ptr(prog->stats);  in update_prog_stats()
885 static void notrace __bpf_prog_exit_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_recur()  argument
891 	update_prog_stats(prog, start);  in __bpf_prog_exit_recur()
892 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_recur()
897 static u64 notrace __bpf_prog_enter_lsm_cgroup(struct bpf_prog *prog,  in __bpf_prog_enter_lsm_cgroup()  argument
912 static void notrace __bpf_prog_exit_lsm_cgroup(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_lsm_cgroup()  argument
922 u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable_recur()  argument
931 	if (unlikely(this_cpu_inc_return(*(prog->active)) != 1)) {  in __bpf_prog_enter_sleepable_recur()
932 		bpf_prog_inc_misses_counter(prog);  in __bpf_prog_enter_sleepable_recur()
938 void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable_recur()  argument
943 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable_recur()
944 	this_cpu_dec(*(prog->active));  in __bpf_prog_exit_sleepable_recur()
949 static u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog,  in __bpf_prog_enter_sleepable()  argument
961 static void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit_sleepable()  argument
966 	update_prog_stats(prog, start);  in __bpf_prog_exit_sleepable()
971 static u64 notrace __bpf_prog_enter(struct bpf_prog *prog,  in __bpf_prog_enter()  argument
983 static void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start,  in __bpf_prog_exit()  argument
989 	update_prog_stats(prog, start);  in __bpf_prog_exit()
1004 bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog)  in bpf_trampoline_enter()  argument
1006 	bool sleepable = prog->aux->sleepable;  in bpf_trampoline_enter()
1008 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_enter()
1012 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_enter()
1013 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_enter()
1019 bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog)  in bpf_trampoline_exit()  argument
1021 	bool sleepable = prog->aux->sleepable;  in bpf_trampoline_exit()
1023 	if (bpf_prog_check_recur(prog))  in bpf_trampoline_exit()
1027 	if (resolve_prog_type(prog) == BPF_PROG_TYPE_LSM &&  in bpf_trampoline_exit()
1028 	    prog->expected_attach_type == BPF_LSM_CGROUP)  in bpf_trampoline_exit()