87 	struct cls_bpf_prog *prog;  in cls_bpf_classify()  local
90 	list_for_each_entry_rcu(prog, &head->plist, link) {  in cls_bpf_classify()
93 		qdisc_skb_cb(skb)->tc_classid = prog->res.classid;  in cls_bpf_classify()
95 		if (tc_skip_sw(prog->gen_flags)) {  in cls_bpf_classify()
96 			filter_res = prog->exts_integrated ? TC_ACT_UNSPEC : 0;  in cls_bpf_classify()
101 			filter_res = bpf_prog_run(prog->filter, skb);  in cls_bpf_classify()
105 			filter_res = bpf_prog_run(prog->filter, skb);  in cls_bpf_classify()
110 		if (prog->exts_integrated) {  in cls_bpf_classify()
112 			res->classid = TC_H_MAJ(prog->res.classid) |  in cls_bpf_classify()
127 			*res = prog->res;  in cls_bpf_classify()
130 		ret = tcf_exts_exec(skb, &prog->exts, res);  in cls_bpf_classify()
140 static bool cls_bpf_is_ebpf(const struct cls_bpf_prog *prog)  in cls_bpf_is_ebpf()  argument
142 	return !prog->bpf_ops;  in cls_bpf_is_ebpf()
145 static int cls_bpf_offload_cmd(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in cls_bpf_offload_cmd()  argument
155 	skip_sw = prog && tc_skip_sw(prog->gen_flags);  in cls_bpf_offload_cmd()
156 	obj = prog ?: oldprog;  in cls_bpf_offload_cmd()
161 	cls_bpf.prog = prog ? prog->filter : NULL;  in cls_bpf_offload_cmd()
166 	if (oldprog && prog)  in cls_bpf_offload_cmd()
170 					  &prog->gen_flags, &prog->in_hw_count,  in cls_bpf_offload_cmd()
172 	else if (prog)  in cls_bpf_offload_cmd()
174 				      skip_sw, &prog->gen_flags,  in cls_bpf_offload_cmd()
175 				      &prog->in_hw_count, true);  in cls_bpf_offload_cmd()
181 	if (prog && err) {  in cls_bpf_offload_cmd()
182 		cls_bpf_offload_cmd(tp, oldprog, prog, extack);  in cls_bpf_offload_cmd()
186 	if (prog && skip_sw && !(prog->gen_flags & TCA_CLS_FLAGS_IN_HW))  in cls_bpf_offload_cmd()
197 static int cls_bpf_offload(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in cls_bpf_offload()  argument
201 	if (prog && oldprog &&  in cls_bpf_offload()
202 	    cls_bpf_flags(prog->gen_flags) !=  in cls_bpf_offload()
206 	if (prog && tc_skip_hw(prog->gen_flags))  in cls_bpf_offload()
207 		prog = NULL;  in cls_bpf_offload()
210 	if (!prog && !oldprog)  in cls_bpf_offload()
213 	return cls_bpf_offload_cmd(tp, prog, oldprog, extack);  in cls_bpf_offload()
217 				 struct cls_bpf_prog *prog,  in cls_bpf_stop_offload()  argument
222 	err = cls_bpf_offload_cmd(tp, NULL, prog, extack);  in cls_bpf_stop_offload()
228 					 struct cls_bpf_prog *prog)  in cls_bpf_offload_update_stats()  argument
233 	tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags, NULL);  in cls_bpf_offload_update_stats()
235 	cls_bpf.exts = &prog->exts;  in cls_bpf_offload_update_stats()
236 	cls_bpf.prog = prog->filter;  in cls_bpf_offload_update_stats()
237 	cls_bpf.name = prog->bpf_name;  in cls_bpf_offload_update_stats()
238 	cls_bpf.exts_integrated = prog->exts_integrated;  in cls_bpf_offload_update_stats()
258 static void cls_bpf_free_parms(struct cls_bpf_prog *prog)  in cls_bpf_free_parms()  argument
260 	if (cls_bpf_is_ebpf(prog))  in cls_bpf_free_parms()
261 		bpf_prog_put(prog->filter);  in cls_bpf_free_parms()
263 		bpf_prog_destroy(prog->filter);  in cls_bpf_free_parms()
265 	kfree(prog->bpf_name);  in cls_bpf_free_parms()
266 	kfree(prog->bpf_ops);  in cls_bpf_free_parms()
269 static void __cls_bpf_delete_prog(struct cls_bpf_prog *prog)  in __cls_bpf_delete_prog()  argument
271 	tcf_exts_destroy(&prog->exts);  in __cls_bpf_delete_prog()
272 	tcf_exts_put_net(&prog->exts);  in __cls_bpf_delete_prog()
274 	cls_bpf_free_parms(prog);  in __cls_bpf_delete_prog()
275 	kfree(prog);  in __cls_bpf_delete_prog()
280 	struct cls_bpf_prog *prog = container_of(to_rcu_work(work),  in cls_bpf_delete_prog_work()  local
284 	__cls_bpf_delete_prog(prog);  in cls_bpf_delete_prog_work()
288 static void __cls_bpf_delete(struct tcf_proto *tp, struct cls_bpf_prog *prog,  in __cls_bpf_delete()  argument
293 	idr_remove(&head->handle_idr, prog->handle);  in __cls_bpf_delete()
294 	cls_bpf_stop_offload(tp, prog, extack);  in __cls_bpf_delete()
295 	list_del_rcu(&prog->link);  in __cls_bpf_delete()
296 	tcf_unbind_filter(tp, &prog->res);  in __cls_bpf_delete()
297 	if (tcf_exts_get_net(&prog->exts))  in __cls_bpf_delete()
298 		tcf_queue_work(&prog->rwork, cls_bpf_delete_prog_work);  in __cls_bpf_delete()
300 		__cls_bpf_delete_prog(prog);  in __cls_bpf_delete()
317 	struct cls_bpf_prog *prog, *tmp;  in cls_bpf_destroy()  local
319 	list_for_each_entry_safe(prog, tmp, &head->plist, link)  in cls_bpf_destroy()
320 		__cls_bpf_delete(tp, prog, extack);  in cls_bpf_destroy()
329 	struct cls_bpf_prog *prog;  in cls_bpf_get()  local
331 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_get()
332 		if (prog->handle == handle)  in cls_bpf_get()
333 			return prog;  in cls_bpf_get()
339 static int cls_bpf_prog_from_ops(struct nlattr **tb, struct cls_bpf_prog *prog)  in cls_bpf_prog_from_ops()  argument
368 	prog->bpf_ops = bpf_ops;  in cls_bpf_prog_from_ops()
369 	prog->bpf_num_ops = bpf_num_ops;  in cls_bpf_prog_from_ops()
370 	prog->bpf_name = NULL;  in cls_bpf_prog_from_ops()
371 	prog->filter = fp;  in cls_bpf_prog_from_ops()
376 static int cls_bpf_prog_from_efd(struct nlattr **tb, struct cls_bpf_prog *prog,  in cls_bpf_prog_from_efd()  argument
399 	prog->bpf_ops = NULL;  in cls_bpf_prog_from_efd()
400 	prog->bpf_name = name;  in cls_bpf_prog_from_efd()
401 	prog->filter = fp;  in cls_bpf_prog_from_efd()
420 	struct cls_bpf_prog *prog;  in cls_bpf_change()  local
432 	prog = kzalloc(sizeof(*prog), GFP_KERNEL);  in cls_bpf_change()
433 	if (!prog)  in cls_bpf_change()
436 	ret = tcf_exts_init(&prog->exts, net, TCA_BPF_ACT, TCA_BPF_POLICE);  in cls_bpf_change()
449 		ret = idr_alloc_u32(&head->handle_idr, prog, &handle,  in cls_bpf_change()
452 		ret = idr_alloc_u32(&head->handle_idr, prog, &handle,  in cls_bpf_change()
458 	prog->handle = handle;  in cls_bpf_change()
467 	ret = tcf_exts_validate(net, tp, tb, tca[TCA_RATE], &prog->exts,  in cls_bpf_change()
491 	prog->exts_integrated = have_exts;  in cls_bpf_change()
492 	prog->gen_flags = gen_flags;  in cls_bpf_change()
494 	ret = is_bpf ? cls_bpf_prog_from_ops(tb, prog) :  in cls_bpf_change()
495 		cls_bpf_prog_from_efd(tb, prog, gen_flags, tp);  in cls_bpf_change()
500 		prog->res.classid = nla_get_u32(tb[TCA_BPF_CLASSID]);  in cls_bpf_change()
501 		tcf_bind_filter(tp, &prog->res, base);  in cls_bpf_change()
505 	ret = cls_bpf_offload(tp, prog, oldprog, extack);  in cls_bpf_change()
509 	if (!tc_in_hw(prog->gen_flags))  in cls_bpf_change()
510 		prog->gen_flags |= TCA_CLS_FLAGS_NOT_IN_HW;  in cls_bpf_change()
513 		idr_replace(&head->handle_idr, prog, handle);  in cls_bpf_change()
514 		list_replace_rcu(&oldprog->link, &prog->link);  in cls_bpf_change()
519 		list_add_rcu(&prog->link, &head->plist);  in cls_bpf_change()
522 	*arg = prog;  in cls_bpf_change()
527 		tcf_unbind_filter(tp, &prog->res);  in cls_bpf_change()
528 	cls_bpf_free_parms(prog);  in cls_bpf_change()
531 		idr_remove(&head->handle_idr, prog->handle);  in cls_bpf_change()
533 	tcf_exts_destroy(&prog->exts);  in cls_bpf_change()
534 	kfree(prog);  in cls_bpf_change()
538 static int cls_bpf_dump_bpf_info(const struct cls_bpf_prog *prog,  in cls_bpf_dump_bpf_info()  argument
543 	if (nla_put_u16(skb, TCA_BPF_OPS_LEN, prog->bpf_num_ops))  in cls_bpf_dump_bpf_info()
546 	nla = nla_reserve(skb, TCA_BPF_OPS, prog->bpf_num_ops *  in cls_bpf_dump_bpf_info()
551 	memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla));  in cls_bpf_dump_bpf_info()
556 static int cls_bpf_dump_ebpf_info(const struct cls_bpf_prog *prog,  in cls_bpf_dump_ebpf_info()  argument
561 	if (prog->bpf_name &&  in cls_bpf_dump_ebpf_info()
562 	    nla_put_string(skb, TCA_BPF_NAME, prog->bpf_name))  in cls_bpf_dump_ebpf_info()
565 	if (nla_put_u32(skb, TCA_BPF_ID, prog->filter->aux->id))  in cls_bpf_dump_ebpf_info()
568 	nla = nla_reserve(skb, TCA_BPF_TAG, sizeof(prog->filter->tag));  in cls_bpf_dump_ebpf_info()
572 	memcpy(nla_data(nla), prog->filter->tag, nla_len(nla));  in cls_bpf_dump_ebpf_info()
580 	struct cls_bpf_prog *prog = fh;  in cls_bpf_dump()  local
585 	if (prog == NULL)  in cls_bpf_dump()
588 	tm->tcm_handle = prog->handle;  in cls_bpf_dump()
590 	cls_bpf_offload_update_stats(tp, prog);  in cls_bpf_dump()
596 	if (prog->res.classid &&  in cls_bpf_dump()
597 	    nla_put_u32(skb, TCA_BPF_CLASSID, prog->res.classid))  in cls_bpf_dump()
600 	if (cls_bpf_is_ebpf(prog))  in cls_bpf_dump()
601 		ret = cls_bpf_dump_ebpf_info(prog, skb);  in cls_bpf_dump()
603 		ret = cls_bpf_dump_bpf_info(prog, skb);  in cls_bpf_dump()
607 	if (tcf_exts_dump(skb, &prog->exts) < 0)  in cls_bpf_dump()
610 	if (prog->exts_integrated)  in cls_bpf_dump()
614 	if (prog->gen_flags &&  in cls_bpf_dump()
615 	    nla_put_u32(skb, TCA_BPF_FLAGS_GEN, prog->gen_flags))  in cls_bpf_dump()
620 	if (tcf_exts_dump_stats(skb, &prog->exts) < 0)  in cls_bpf_dump()
633 	struct cls_bpf_prog *prog = fh;  in cls_bpf_bind_class()  local
635 	tc_cls_bind_class(classid, cl, q, &prog->res, base);  in cls_bpf_bind_class()
642 	struct cls_bpf_prog *prog;  in cls_bpf_walk()  local
644 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_walk()
645 		if (!tc_cls_stats_dump(tp, arg, prog))  in cls_bpf_walk()
656 	struct cls_bpf_prog *prog;  in cls_bpf_reoffload()  local
659 	list_for_each_entry(prog, &head->plist, link) {  in cls_bpf_reoffload()
660 		if (tc_skip_hw(prog->gen_flags))  in cls_bpf_reoffload()
663 		tc_cls_common_offload_init(&cls_bpf.common, tp, prog->gen_flags,  in cls_bpf_reoffload()
666 		cls_bpf.exts = &prog->exts;  in cls_bpf_reoffload()
667 		cls_bpf.prog = add ? prog->filter : NULL;  in cls_bpf_reoffload()
668 		cls_bpf.oldprog = add ? NULL : prog->filter;  in cls_bpf_reoffload()
669 		cls_bpf.name = prog->bpf_name;  in cls_bpf_reoffload()
670 		cls_bpf.exts_integrated = prog->exts_integrated;  in cls_bpf_reoffload()
673 					    &cls_bpf, cb_priv, &prog->gen_flags,  in cls_bpf_reoffload()
674 					    &prog->in_hw_count);  in cls_bpf_reoffload()