20 nfp_bpf_goto_meta(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,  in nfp_bpf_goto_meta()  argument
28 	if (min(forward, backward) > nfp_prog->n_insns - insn_idx - 1) {  in nfp_bpf_goto_meta()
29 		backward = nfp_prog->n_insns - insn_idx - 1;  in nfp_bpf_goto_meta()
30 		meta = nfp_prog_last_meta(nfp_prog);  in nfp_bpf_goto_meta()
34 		meta = nfp_prog_first_meta(nfp_prog);  in nfp_bpf_goto_meta()
48 nfp_record_adjust_head(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,  in nfp_record_adjust_head()  argument
69 	if (nfp_prog->adjust_head_location) {  in nfp_record_adjust_head()
71 		if (nfp_prog->adjust_head_location != meta->n)  in nfp_record_adjust_head()
80 	nfp_prog->adjust_head_location = location;  in nfp_record_adjust_head()
170 nfp_bpf_check_helper_call(struct nfp_prog *nfp_prog,  in nfp_bpf_check_helper_call()  argument
177 	struct nfp_app_bpf *bpf = nfp_prog->bpf;  in nfp_bpf_check_helper_call()
191 		nfp_record_adjust_head(bpf, nfp_prog, meta, reg2);  in nfp_bpf_check_helper_call()
284 		dev_warn_once(&nfp_prog->bpf->app->pf->pdev->dev,  in nfp_bpf_check_helper_call()
311 nfp_bpf_check_exit(struct nfp_prog *nfp_prog,  in nfp_bpf_check_exit()  argument
317 	if (nfp_prog->type == BPF_PROG_TYPE_XDP)  in nfp_bpf_check_exit()
330 	if (nfp_prog->type == BPF_PROG_TYPE_SCHED_CLS &&  in nfp_bpf_check_exit()
343 nfp_bpf_check_stack_access(struct nfp_prog *nfp_prog,  in nfp_bpf_check_stack_access()  argument
451 nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,  in nfp_bpf_check_ptr()  argument
466 		err = nfp_bpf_check_stack_access(nfp_prog, meta, reg, env);  in nfp_bpf_check_ptr()
502 nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,  in nfp_bpf_check_store()  argument
508 		if (nfp_prog->type == BPF_PROG_TYPE_XDP) {  in nfp_bpf_check_store()
512 				if (nfp_prog->bpf->queue_select)  in nfp_bpf_check_store()
522 	return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);  in nfp_bpf_check_store()
526 nfp_bpf_check_atomic(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,  in nfp_bpf_check_atomic()  argument
552 	return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg);  in nfp_bpf_check_atomic()
556 nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,  in nfp_bpf_check_alu()  argument
634 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;  in nfp_verify_insn()  local
635 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;  in nfp_verify_insn()
637 	meta = nfp_bpf_goto_meta(nfp_prog, meta, insn_idx);  in nfp_verify_insn()
638 	nfp_prog->verifier_meta = meta;  in nfp_verify_insn()
653 		return nfp_bpf_check_helper_call(nfp_prog, env, meta);  in nfp_verify_insn()
655 		return nfp_bpf_check_exit(nfp_prog, env);  in nfp_verify_insn()
658 		return nfp_bpf_check_ptr(nfp_prog, meta, env,  in nfp_verify_insn()
661 		return nfp_bpf_check_store(nfp_prog, meta, env);  in nfp_verify_insn()
664 		return nfp_bpf_check_atomic(nfp_prog, meta, env);  in nfp_verify_insn()
667 		return nfp_bpf_check_alu(nfp_prog, meta, env);  in nfp_verify_insn()
674 				struct nfp_prog *nfp_prog)  in nfp_assign_subprog_idx_and_regs()  argument
679 	list_for_each_entry(meta, &nfp_prog->insns, l) {  in nfp_assign_subprog_idx_and_regs()
686 			nfp_prog->subprog[index].needs_reg_push = 1;  in nfp_assign_subprog_idx_and_regs()
689 	if (index + 1 != nfp_prog->subprog_cnt) {  in nfp_assign_subprog_idx_and_regs()
691 			index + 1, nfp_prog->subprog_cnt);  in nfp_assign_subprog_idx_and_regs()
698 static unsigned int nfp_bpf_get_stack_usage(struct nfp_prog *nfp_prog)  in nfp_bpf_get_stack_usage()  argument
700 	struct nfp_insn_meta *meta = nfp_prog_first_meta(nfp_prog);  in nfp_bpf_get_stack_usage()
714 	frame_depths[frame] = nfp_prog->subprog[idx].stack_depth;  in nfp_bpf_get_stack_usage()
720 	for (; meta != nfp_prog_last_meta(nfp_prog) && meta->subprog_idx == idx;  in nfp_bpf_get_stack_usage()
732 		meta = nfp_bpf_goto_meta(nfp_prog, meta,  in nfp_bpf_get_stack_usage()
752 static void nfp_bpf_insn_flag_zext(struct nfp_prog *nfp_prog,  in nfp_bpf_insn_flag_zext()  argument
757 	list_for_each_entry(meta, &nfp_prog->insns, l) {  in nfp_bpf_insn_flag_zext()
766 	struct nfp_prog *nfp_prog;  in nfp_bpf_finalize()  local
771 	nfp_prog = env->prog->aux->offload->dev_priv;  in nfp_bpf_finalize()
772 	nfp_prog->subprog_cnt = env->subprog_cnt;  in nfp_bpf_finalize()
773 	nfp_prog->subprog = kcalloc(nfp_prog->subprog_cnt,  in nfp_bpf_finalize()
774 				    sizeof(nfp_prog->subprog[0]), GFP_KERNEL);  in nfp_bpf_finalize()
775 	if (!nfp_prog->subprog)  in nfp_bpf_finalize()
778 	nfp_assign_subprog_idx_and_regs(env, nfp_prog);  in nfp_bpf_finalize()
781 	for (i = 0; i < nfp_prog->subprog_cnt; i++) {  in nfp_bpf_finalize()
782 		nfp_prog->subprog[i].stack_depth = info[i].stack_depth;  in nfp_bpf_finalize()
788 		nfp_prog->subprog[i].stack_depth += REG_WIDTH;  in nfp_bpf_finalize()
790 		if (nfp_prog->subprog[i].needs_reg_push)  in nfp_bpf_finalize()
791 			nfp_prog->subprog[i].stack_depth += BPF_REG_SIZE * 4;  in nfp_bpf_finalize()
796 	nfp_prog->stack_size = nfp_bpf_get_stack_usage(nfp_prog);  in nfp_bpf_finalize()
797 	if (nfp_prog->stack_size > max_stack) {  in nfp_bpf_finalize()
799 			nfp_prog->stack_size, max_stack);  in nfp_bpf_finalize()
803 	nfp_bpf_insn_flag_zext(nfp_prog, env->insn_aux_data);  in nfp_bpf_finalize()
810 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;  in nfp_bpf_opt_replace_insn()  local
812 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;  in nfp_bpf_opt_replace_insn()
814 	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);  in nfp_bpf_opt_replace_insn()
815 	nfp_prog->verifier_meta = meta;  in nfp_bpf_opt_replace_insn()
843 	struct nfp_prog *nfp_prog = env->prog->aux->offload->dev_priv;  in nfp_bpf_opt_remove_insns()  local
845 	struct nfp_insn_meta *meta = nfp_prog->verifier_meta;  in nfp_bpf_opt_remove_insns()
848 	meta = nfp_bpf_goto_meta(nfp_prog, meta, aux_data[off].orig_idx);  in nfp_bpf_opt_remove_insns()
851 		if (WARN_ON_ONCE(&meta->l == &nfp_prog->insns))  in nfp_bpf_opt_remove_insns()