1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 21da177e4SLinus Torvalds #ifndef __NET_PKT_CLS_H 31da177e4SLinus Torvalds #define __NET_PKT_CLS_H 41da177e4SLinus Torvalds 51da177e4SLinus Torvalds #include <linux/pkt_cls.h> 67aa0045dSCong Wang #include <linux/workqueue.h> 71da177e4SLinus Torvalds #include <net/sch_generic.h> 81da177e4SLinus Torvalds #include <net/act_api.h> 9a5148626SJiri Pirko #include <net/net_namespace.h> 101da177e4SLinus Torvalds 11cd11b164SPaolo Abeni /* TC action not accessible from user space */ 12720f22feSJohn Hurley #define TC_ACT_CONSUMED (TC_ACT_VALUE_MAX + 1) 13cd11b164SPaolo Abeni 141da177e4SLinus Torvalds /* Basic packet classifier frontend definitions. */ 151da177e4SLinus Torvalds 16fd2c3ef7SEric Dumazet struct tcf_walker { 171da177e4SLinus Torvalds int stop; 181da177e4SLinus Torvalds int skip; 191da177e4SLinus Torvalds int count; 206676d5e4SVlad Buslov bool nonempty; 2101683a14SVlad Buslov unsigned long cookie; 228113c095SWANG Cong int (*fn)(struct tcf_proto *, void *node, struct tcf_walker *); 231da177e4SLinus Torvalds }; 241da177e4SLinus Torvalds 255c15257fSJoe Perches int register_tcf_proto_ops(struct tcf_proto_ops *ops); 26bc5c8260SZhengchao Shao void unregister_tcf_proto_ops(struct tcf_proto_ops *ops); 271da177e4SLinus Torvalds 288c4083b3SJiri Pirko struct tcf_block_ext_info { 2932f8c409SPablo Neira Ayuso enum flow_block_binder_type binder_type; 30c7eb7d72SJiri Pirko tcf_chain_head_change_t *chain_head_change; 31c7eb7d72SJiri Pirko void *chain_head_change_priv; 3248617387SJiri Pirko u32 block_index; 338c4083b3SJiri Pirko }; 348c4083b3SJiri Pirko 353625750fSPetr Machata struct tcf_qevent { 363625750fSPetr Machata struct tcf_block *block; 373625750fSPetr Machata struct tcf_block_ext_info info; 383625750fSPetr Machata struct tcf_proto __rcu *filter_chain; 393625750fSPetr Machata }; 403625750fSPetr Machata 41acb67442SJiri Pirko struct tcf_block_cb; 42aaa908ffSCong Wang bool tcf_queue_work(struct rcu_work *rwork, work_func_t func); 43acb67442SJiri Pirko 448ae70032SJiri Pirko #ifdef CONFIG_NET_CLS 451f3ed383SJiri Pirko struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, 461f3ed383SJiri Pirko u32 chain_index); 471f3ed383SJiri Pirko void tcf_chain_put_by_act(struct tcf_chain *chain); 48bbf73830SVlad Buslov struct tcf_chain *tcf_get_next_chain(struct tcf_block *block, 49bbf73830SVlad Buslov struct tcf_chain *chain); 50fe2923afSVlad Buslov struct tcf_proto *tcf_get_next_proto(struct tcf_chain *chain, 510fca55edSVlad Buslov struct tcf_proto *tp); 52f36fe1c4SJiri Pirko void tcf_block_netif_keep_dst(struct tcf_block *block); 536529eabaSJiri Pirko int tcf_block_get(struct tcf_block **p_block, 548d1a77f9SAlexander Aring struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 558d1a77f9SAlexander Aring struct netlink_ext_ack *extack); 56c7eb7d72SJiri Pirko int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 578d1a77f9SAlexander Aring struct tcf_block_ext_info *ei, 588d1a77f9SAlexander Aring struct netlink_ext_ack *extack); 596529eabaSJiri Pirko void tcf_block_put(struct tcf_block *block); 60c7eb7d72SJiri Pirko void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 618c4083b3SJiri Pirko struct tcf_block_ext_info *ei); 6280cd22c3SPaul Blakey int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, 6380cd22c3SPaul Blakey int police, struct tcf_proto *tp, u32 handle, bool used_action_miss); 6444186460SJiri Pirko 6548617387SJiri Pirko static inline bool tcf_block_shared(struct tcf_block *block) 6648617387SJiri Pirko { 6748617387SJiri Pirko return block->index; 6848617387SJiri Pirko } 6948617387SJiri Pirko 70c1a970d0SVlad Buslov static inline bool tcf_block_non_null_shared(struct tcf_block *block) 71c1a970d0SVlad Buslov { 72c1a970d0SVlad Buslov return block && block->index; 73c1a970d0SVlad Buslov } 74c1a970d0SVlad Buslov 7544186460SJiri Pirko static inline struct Qdisc *tcf_block_q(struct tcf_block *block) 7644186460SJiri Pirko { 7748617387SJiri Pirko WARN_ON(tcf_block_shared(block)); 7844186460SJiri Pirko return block->q; 7944186460SJiri Pirko } 8044186460SJiri Pirko 813aa26055SDavide Caratti int tcf_classify(struct sk_buff *skb, 823aa26055SDavide Caratti const struct tcf_block *block, 837d17c544SPaul Blakey const struct tcf_proto *tp, struct tcf_result *res, 847d17c544SPaul Blakey bool compat_mode); 8587d83093SJiri Pirko 86fe0df81dSZhengchao Shao static inline bool tc_cls_stats_dump(struct tcf_proto *tp, 87fe0df81dSZhengchao Shao struct tcf_walker *arg, 88fe0df81dSZhengchao Shao void *filter) 89fe0df81dSZhengchao Shao { 90fe0df81dSZhengchao Shao if (arg->count >= arg->skip && arg->fn(tp, filter, arg) < 0) { 91fe0df81dSZhengchao Shao arg->stop = 1; 92fe0df81dSZhengchao Shao return false; 93fe0df81dSZhengchao Shao } 94fe0df81dSZhengchao Shao 95fe0df81dSZhengchao Shao arg->count++; 96fe0df81dSZhengchao Shao return true; 97fe0df81dSZhengchao Shao } 98fe0df81dSZhengchao Shao 998ae70032SJiri Pirko #else 10088c44a52SPieter Jansen van Vuuren static inline bool tcf_block_shared(struct tcf_block *block) 10188c44a52SPieter Jansen van Vuuren { 10288c44a52SPieter Jansen van Vuuren return false; 10388c44a52SPieter Jansen van Vuuren } 10488c44a52SPieter Jansen van Vuuren 105c1a970d0SVlad Buslov static inline bool tcf_block_non_null_shared(struct tcf_block *block) 106c1a970d0SVlad Buslov { 107c1a970d0SVlad Buslov return false; 108c1a970d0SVlad Buslov } 109c1a970d0SVlad Buslov 1106529eabaSJiri Pirko static inline 1116529eabaSJiri Pirko int tcf_block_get(struct tcf_block **p_block, 1123c149091SSudip Mukherjee struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 1133c149091SSudip Mukherjee struct netlink_ext_ack *extack) 1146529eabaSJiri Pirko { 1156529eabaSJiri Pirko return 0; 1166529eabaSJiri Pirko } 1176529eabaSJiri Pirko 1188c4083b3SJiri Pirko static inline 119c7eb7d72SJiri Pirko int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 12033c30a8bSQuentin Monnet struct tcf_block_ext_info *ei, 12133c30a8bSQuentin Monnet struct netlink_ext_ack *extack) 1228c4083b3SJiri Pirko { 1238c4083b3SJiri Pirko return 0; 1248c4083b3SJiri Pirko } 1258c4083b3SJiri Pirko 1266529eabaSJiri Pirko static inline void tcf_block_put(struct tcf_block *block) 1278ae70032SJiri Pirko { 1288ae70032SJiri Pirko } 12987d83093SJiri Pirko 1308c4083b3SJiri Pirko static inline 131c7eb7d72SJiri Pirko void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 1328c4083b3SJiri Pirko struct tcf_block_ext_info *ei) 1338c4083b3SJiri Pirko { 1348c4083b3SJiri Pirko } 1358c4083b3SJiri Pirko 13644186460SJiri Pirko static inline struct Qdisc *tcf_block_q(struct tcf_block *block) 13744186460SJiri Pirko { 13844186460SJiri Pirko return NULL; 13944186460SJiri Pirko } 14044186460SJiri Pirko 141acb67442SJiri Pirko static inline 142a7323311SPablo Neira Ayuso int tc_setup_cb_block_register(struct tcf_block *block, flow_setup_cb_t *cb, 143acb67442SJiri Pirko void *cb_priv) 144acb67442SJiri Pirko { 145acb67442SJiri Pirko return 0; 146acb67442SJiri Pirko } 147acb67442SJiri Pirko 148acb67442SJiri Pirko static inline 149a7323311SPablo Neira Ayuso void tc_setup_cb_block_unregister(struct tcf_block *block, flow_setup_cb_t *cb, 150acb67442SJiri Pirko void *cb_priv) 151acb67442SJiri Pirko { 152acb67442SJiri Pirko } 153acb67442SJiri Pirko 1543aa26055SDavide Caratti static inline int tcf_classify(struct sk_buff *skb, 1553aa26055SDavide Caratti const struct tcf_block *block, 1569410c940SPaul Blakey const struct tcf_proto *tp, 1579410c940SPaul Blakey struct tcf_result *res, bool compat_mode) 1589410c940SPaul Blakey { 1599410c940SPaul Blakey return TC_ACT_UNSPEC; 1609410c940SPaul Blakey } 1619410c940SPaul Blakey 1628ae70032SJiri Pirko #endif 163cf1facdaSJiri Pirko 1641da177e4SLinus Torvalds static inline unsigned long 1651da177e4SLinus Torvalds __cls_set_class(unsigned long *clp, unsigned long cl) 1661da177e4SLinus Torvalds { 167a0efb80cSWANG Cong return xchg(clp, cl); 1681da177e4SLinus Torvalds } 1691da177e4SLinus Torvalds 1702e24cd75SCong Wang static inline void 1712e24cd75SCong Wang __tcf_bind_filter(struct Qdisc *q, struct tcf_result *r, unsigned long base) 1721da177e4SLinus Torvalds { 1732e24cd75SCong Wang unsigned long cl; 1741da177e4SLinus Torvalds 1752e24cd75SCong Wang cl = q->ops->cl_ops->bind_tcf(q, base, r->classid); 1762e24cd75SCong Wang cl = __cls_set_class(&r->class, cl); 1772e24cd75SCong Wang if (cl) 1782e24cd75SCong Wang q->ops->cl_ops->unbind_tcf(q, cl); 1791da177e4SLinus Torvalds } 1801da177e4SLinus Torvalds 1811da177e4SLinus Torvalds static inline void 1821da177e4SLinus Torvalds tcf_bind_filter(struct tcf_proto *tp, struct tcf_result *r, unsigned long base) 1831da177e4SLinus Torvalds { 18434e3759cSJiri Pirko struct Qdisc *q = tp->chain->block->q; 1851da177e4SLinus Torvalds 18634e3759cSJiri Pirko /* Check q as it is not set for shared blocks. In that case, 18734e3759cSJiri Pirko * setting class is not supported. 18834e3759cSJiri Pirko */ 18934e3759cSJiri Pirko if (!q) 19034e3759cSJiri Pirko return; 1912e24cd75SCong Wang sch_tree_lock(q); 1922e24cd75SCong Wang __tcf_bind_filter(q, r, base); 1932e24cd75SCong Wang sch_tree_unlock(q); 1942e24cd75SCong Wang } 1952e24cd75SCong Wang 1962e24cd75SCong Wang static inline void 1972e24cd75SCong Wang __tcf_unbind_filter(struct Qdisc *q, struct tcf_result *r) 1982e24cd75SCong Wang { 1992e24cd75SCong Wang unsigned long cl; 2002e24cd75SCong Wang 2012e24cd75SCong Wang if ((cl = __cls_set_class(&r->class, 0)) != 0) 20234e3759cSJiri Pirko q->ops->cl_ops->unbind_tcf(q, cl); 2031da177e4SLinus Torvalds } 2041da177e4SLinus Torvalds 2051da177e4SLinus Torvalds static inline void 2061da177e4SLinus Torvalds tcf_unbind_filter(struct tcf_proto *tp, struct tcf_result *r) 2071da177e4SLinus Torvalds { 20834e3759cSJiri Pirko struct Qdisc *q = tp->chain->block->q; 2091da177e4SLinus Torvalds 21034e3759cSJiri Pirko if (!q) 21134e3759cSJiri Pirko return; 2122e24cd75SCong Wang __tcf_unbind_filter(q, r); 2131da177e4SLinus Torvalds } 2141da177e4SLinus Torvalds 215402963e3SZhengchao Shao static inline void tc_cls_bind_class(u32 classid, unsigned long cl, 216402963e3SZhengchao Shao void *q, struct tcf_result *res, 217402963e3SZhengchao Shao unsigned long base) 218402963e3SZhengchao Shao { 219402963e3SZhengchao Shao if (res->classid == classid) { 220402963e3SZhengchao Shao if (cl) 221402963e3SZhengchao Shao __tcf_bind_filter(q, res, base); 222402963e3SZhengchao Shao else 223402963e3SZhengchao Shao __tcf_unbind_filter(q, res); 224402963e3SZhengchao Shao } 225402963e3SZhengchao Shao } 226402963e3SZhengchao Shao 227fd2c3ef7SEric Dumazet struct tcf_exts { 2281da177e4SLinus Torvalds #ifdef CONFIG_NET_CLS_ACT 22933be6271SWANG Cong __u32 type; /* for backward compat(TCA_OLD_COMPAT) */ 23022dc13c8SWANG Cong int nr_actions; 23122dc13c8SWANG Cong struct tc_action **actions; 232e4b95c41SCong Wang struct net *net; 233dbdcda63SEric Dumazet netns_tracker ns_tracker; 23480cd22c3SPaul Blakey struct tcf_exts_miss_cookie_node *miss_cookie_node; 2351da177e4SLinus Torvalds #endif 2361da177e4SLinus Torvalds /* Map to export classifier specific extension TLV types to the 2371da177e4SLinus Torvalds * generic extensions API. Unsupported extensions must be set to 0. 2381da177e4SLinus Torvalds */ 2391da177e4SLinus Torvalds int action; 2401da177e4SLinus Torvalds int police; 2411da177e4SLinus Torvalds }; 2421da177e4SLinus Torvalds 24314215108SCong Wang static inline int tcf_exts_init(struct tcf_exts *exts, struct net *net, 24414215108SCong Wang int action, int police) 24533be6271SWANG Cong { 24680cd22c3SPaul Blakey #ifdef CONFIG_NET_CLS 24780cd22c3SPaul Blakey return tcf_exts_init_ex(exts, net, action, police, NULL, 0, false); 24880cd22c3SPaul Blakey #else 24980cd22c3SPaul Blakey return -EOPNOTSUPP; 25033be6271SWANG Cong #endif 25133be6271SWANG Cong } 25233be6271SWANG Cong 253e4b95c41SCong Wang /* Return false if the netns is being destroyed in cleanup_net(). Callers 254e4b95c41SCong Wang * need to do cleanup synchronously in this case, otherwise may race with 255e4b95c41SCong Wang * tc_action_net_exit(). Return true for other cases. 256e4b95c41SCong Wang */ 257e4b95c41SCong Wang static inline bool tcf_exts_get_net(struct tcf_exts *exts) 258e4b95c41SCong Wang { 259e4b95c41SCong Wang #ifdef CONFIG_NET_CLS_ACT 260e4b95c41SCong Wang exts->net = maybe_get_net(exts->net); 261dbdcda63SEric Dumazet if (exts->net) 262dbdcda63SEric Dumazet netns_tracker_alloc(exts->net, &exts->ns_tracker, GFP_KERNEL); 263e4b95c41SCong Wang return exts->net != NULL; 264e4b95c41SCong Wang #else 265e4b95c41SCong Wang return true; 266e4b95c41SCong Wang #endif 267e4b95c41SCong Wang } 268e4b95c41SCong Wang 269e4b95c41SCong Wang static inline void tcf_exts_put_net(struct tcf_exts *exts) 270e4b95c41SCong Wang { 271e4b95c41SCong Wang #ifdef CONFIG_NET_CLS_ACT 272e4b95c41SCong Wang if (exts->net) 273dbdcda63SEric Dumazet put_net_track(exts->net, &exts->ns_tracker); 274e4b95c41SCong Wang #endif 275e4b95c41SCong Wang } 276e4b95c41SCong Wang 27722dc13c8SWANG Cong #ifdef CONFIG_NET_CLS_ACT 278244cd96aSCong Wang #define tcf_exts_for_each_action(i, a, exts) \ 279244cd96aSCong Wang for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = (exts)->actions[i]); i++) 280244cd96aSCong Wang #else 281244cd96aSCong Wang #define tcf_exts_for_each_action(i, a, exts) \ 282191672caSArnd Bergmann for (; 0; (void)(i), (void)(a), (void)(exts)) 28322dc13c8SWANG Cong #endif 28422dc13c8SWANG Cong 2858cbfe939SBaowen Zheng #define tcf_act_for_each_action(i, a, actions) \ 2868cbfe939SBaowen Zheng for (i = 0; i < TCA_ACT_MAX_PRIO && ((a) = actions[i]); i++) 2878cbfe939SBaowen Zheng 2885246c896SOz Shlomo static inline bool tc_act_in_hw(struct tc_action *act) 2895246c896SOz Shlomo { 2905246c896SOz Shlomo return !!act->in_hw_count; 2915246c896SOz Shlomo } 2925246c896SOz Shlomo 293d897a638SJakub Kicinski static inline void 294bcd64368SBaowen Zheng tcf_exts_hw_stats_update(const struct tcf_exts *exts, 2955246c896SOz Shlomo struct flow_stats *stats, 2965246c896SOz Shlomo bool use_act_stats) 297d897a638SJakub Kicinski { 298d897a638SJakub Kicinski #ifdef CONFIG_NET_CLS_ACT 299d897a638SJakub Kicinski int i; 300d897a638SJakub Kicinski 301d897a638SJakub Kicinski for (i = 0; i < exts->nr_actions; i++) { 302d897a638SJakub Kicinski struct tc_action *a = exts->actions[i]; 303d897a638SJakub Kicinski 3045246c896SOz Shlomo if (use_act_stats || tc_act_in_hw(a)) { 3055246c896SOz Shlomo if (!tcf_action_update_hw_stats(a)) 3065246c896SOz Shlomo continue; 3075246c896SOz Shlomo } 3085246c896SOz Shlomo 309c7a66f8dSBaowen Zheng preempt_disable(); 310ac7d2790SOz Shlomo tcf_action_stats_update(a, stats->bytes, stats->pkts, stats->drops, 311ac7d2790SOz Shlomo stats->lastused, true); 312c7a66f8dSBaowen Zheng preempt_enable(); 313c7a66f8dSBaowen Zheng 314ac7d2790SOz Shlomo a->used_hw_stats = stats->used_hw_stats; 315ac7d2790SOz Shlomo a->used_hw_stats_valid = stats->used_hw_stats_valid; 316d897a638SJakub Kicinski } 317d897a638SJakub Kicinski #endif 318d897a638SJakub Kicinski } 319d897a638SJakub Kicinski 3201da177e4SLinus Torvalds /** 3213bcc0cecSJiri Pirko * tcf_exts_has_actions - check if at least one action is present 3223bcc0cecSJiri Pirko * @exts: tc filter extensions handle 3233bcc0cecSJiri Pirko * 3243bcc0cecSJiri Pirko * Returns true if at least one action is present. 3253bcc0cecSJiri Pirko */ 3263bcc0cecSJiri Pirko static inline bool tcf_exts_has_actions(struct tcf_exts *exts) 3273bcc0cecSJiri Pirko { 3282734437eSWANG Cong #ifdef CONFIG_NET_CLS_ACT 3293bcc0cecSJiri Pirko return exts->nr_actions; 3303bcc0cecSJiri Pirko #else 3313bcc0cecSJiri Pirko return false; 3323bcc0cecSJiri Pirko #endif 3333bcc0cecSJiri Pirko } 3342734437eSWANG Cong 3353bcc0cecSJiri Pirko /** 336af69afc5SJiri Pirko * tcf_exts_exec - execute tc filter extensions 337af69afc5SJiri Pirko * @skb: socket buffer 338af69afc5SJiri Pirko * @exts: tc filter extensions handle 339af69afc5SJiri Pirko * @res: desired result 340af69afc5SJiri Pirko * 341af089e70SJiri Pirko * Executes all configured extensions. Returns TC_ACT_OK on a normal execution, 342af69afc5SJiri Pirko * a negative number if the filter must be considered unmatched or 343af69afc5SJiri Pirko * a positive action code (TC_ACT_*) which must be returned to the 344af69afc5SJiri Pirko * underlying layer. 345af69afc5SJiri Pirko */ 346af69afc5SJiri Pirko static inline int 347af69afc5SJiri Pirko tcf_exts_exec(struct sk_buff *skb, struct tcf_exts *exts, 348af69afc5SJiri Pirko struct tcf_result *res) 349af69afc5SJiri Pirko { 350af69afc5SJiri Pirko #ifdef CONFIG_NET_CLS_ACT 351ec1a9ccaSJiri Pirko return tcf_action_exec(skb, exts->actions, exts->nr_actions, res); 352af69afc5SJiri Pirko #endif 353af089e70SJiri Pirko return TC_ACT_OK; 354af69afc5SJiri Pirko } 355af69afc5SJiri Pirko 35680cd22c3SPaul Blakey static inline int 35780cd22c3SPaul Blakey tcf_exts_exec_ex(struct sk_buff *skb, struct tcf_exts *exts, int act_index, 35880cd22c3SPaul Blakey struct tcf_result *res) 35980cd22c3SPaul Blakey { 36080cd22c3SPaul Blakey #ifdef CONFIG_NET_CLS_ACT 36180cd22c3SPaul Blakey return tcf_action_exec(skb, exts->actions + act_index, 36280cd22c3SPaul Blakey exts->nr_actions - act_index, res); 36380cd22c3SPaul Blakey #else 36480cd22c3SPaul Blakey return TC_ACT_OK; 36580cd22c3SPaul Blakey #endif 36680cd22c3SPaul Blakey } 36780cd22c3SPaul Blakey 3685c15257fSJoe Perches int tcf_exts_validate(struct net *net, struct tcf_proto *tp, 369c1b52739SBenjamin LaHaise struct nlattr **tb, struct nlattr *rate_tlv, 370695176bfSCong Wang struct tcf_exts *exts, u32 flags, 37150a56190SAlexander Aring struct netlink_ext_ack *extack); 372c86e0209SBaowen Zheng int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 373c86e0209SBaowen Zheng struct nlattr *rate_tlv, struct tcf_exts *exts, 374c86e0209SBaowen Zheng u32 flags, u32 fl_flags, struct netlink_ext_ack *extack); 37518d0264fSWANG Cong void tcf_exts_destroy(struct tcf_exts *exts); 3769b0d4446SJiri Pirko void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src); 3775da57f42SWANG Cong int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts); 378ca44b738SVlad Buslov int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts); 3795da57f42SWANG Cong int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts); 3801da177e4SLinus Torvalds 3811da177e4SLinus Torvalds /** 3821da177e4SLinus Torvalds * struct tcf_pkt_info - packet information 3830161d151SBijie Xu * 3840161d151SBijie Xu * @ptr: start of the pkt data 3850161d151SBijie Xu * @nexthdr: offset of the next header 3861da177e4SLinus Torvalds */ 387fd2c3ef7SEric Dumazet struct tcf_pkt_info { 3881da177e4SLinus Torvalds unsigned char * ptr; 3891da177e4SLinus Torvalds int nexthdr; 3901da177e4SLinus Torvalds }; 3911da177e4SLinus Torvalds 3921da177e4SLinus Torvalds #ifdef CONFIG_NET_EMATCH 3931da177e4SLinus Torvalds 3941da177e4SLinus Torvalds struct tcf_ematch_ops; 3951da177e4SLinus Torvalds 3961da177e4SLinus Torvalds /** 3971da177e4SLinus Torvalds * struct tcf_ematch - extended match (ematch) 3981da177e4SLinus Torvalds * 3991da177e4SLinus Torvalds * @matchid: identifier to allow userspace to reidentify a match 4001da177e4SLinus Torvalds * @flags: flags specifying attributes and the relation to other matches 4011da177e4SLinus Torvalds * @ops: the operations lookup table of the corresponding ematch module 4021da177e4SLinus Torvalds * @datalen: length of the ematch specific configuration data 4031da177e4SLinus Torvalds * @data: ematch specific data 4040161d151SBijie Xu * @net: the network namespace 4051da177e4SLinus Torvalds */ 406fd2c3ef7SEric Dumazet struct tcf_ematch { 4071da177e4SLinus Torvalds struct tcf_ematch_ops * ops; 4081da177e4SLinus Torvalds unsigned long data; 4091da177e4SLinus Torvalds unsigned int datalen; 4101da177e4SLinus Torvalds u16 matchid; 4111da177e4SLinus Torvalds u16 flags; 41282a470f1SJohn Fastabend struct net *net; 4131da177e4SLinus Torvalds }; 4141da177e4SLinus Torvalds 4151da177e4SLinus Torvalds static inline int tcf_em_is_container(struct tcf_ematch *em) 4161da177e4SLinus Torvalds { 4171da177e4SLinus Torvalds return !em->ops; 4181da177e4SLinus Torvalds } 4191da177e4SLinus Torvalds 4201da177e4SLinus Torvalds static inline int tcf_em_is_simple(struct tcf_ematch *em) 4211da177e4SLinus Torvalds { 4221da177e4SLinus Torvalds return em->flags & TCF_EM_SIMPLE; 4231da177e4SLinus Torvalds } 4241da177e4SLinus Torvalds 4251da177e4SLinus Torvalds static inline int tcf_em_is_inverted(struct tcf_ematch *em) 4261da177e4SLinus Torvalds { 4271da177e4SLinus Torvalds return em->flags & TCF_EM_INVERT; 4281da177e4SLinus Torvalds } 4291da177e4SLinus Torvalds 4301da177e4SLinus Torvalds static inline int tcf_em_last_match(struct tcf_ematch *em) 4311da177e4SLinus Torvalds { 4321da177e4SLinus Torvalds return (em->flags & TCF_EM_REL_MASK) == TCF_EM_REL_END; 4331da177e4SLinus Torvalds } 4341da177e4SLinus Torvalds 4351da177e4SLinus Torvalds static inline int tcf_em_early_end(struct tcf_ematch *em, int result) 4361da177e4SLinus Torvalds { 4371da177e4SLinus Torvalds if (tcf_em_last_match(em)) 4381da177e4SLinus Torvalds return 1; 4391da177e4SLinus Torvalds 4401da177e4SLinus Torvalds if (result == 0 && em->flags & TCF_EM_REL_AND) 4411da177e4SLinus Torvalds return 1; 4421da177e4SLinus Torvalds 4431da177e4SLinus Torvalds if (result != 0 && em->flags & TCF_EM_REL_OR) 4441da177e4SLinus Torvalds return 1; 4451da177e4SLinus Torvalds 4461da177e4SLinus Torvalds return 0; 4471da177e4SLinus Torvalds } 4481da177e4SLinus Torvalds 4491da177e4SLinus Torvalds /** 4501da177e4SLinus Torvalds * struct tcf_ematch_tree - ematch tree handle 4511da177e4SLinus Torvalds * 4521da177e4SLinus Torvalds * @hdr: ematch tree header supplied by userspace 4531da177e4SLinus Torvalds * @matches: array of ematches 4541da177e4SLinus Torvalds */ 455fd2c3ef7SEric Dumazet struct tcf_ematch_tree { 4561da177e4SLinus Torvalds struct tcf_ematch_tree_hdr hdr; 4571da177e4SLinus Torvalds struct tcf_ematch * matches; 4581da177e4SLinus Torvalds 4591da177e4SLinus Torvalds }; 4601da177e4SLinus Torvalds 4611da177e4SLinus Torvalds /** 4621da177e4SLinus Torvalds * struct tcf_ematch_ops - ematch module operations 4631da177e4SLinus Torvalds * 4641da177e4SLinus Torvalds * @kind: identifier (kind) of this ematch module 4651da177e4SLinus Torvalds * @datalen: length of expected configuration data (optional) 4661da177e4SLinus Torvalds * @change: called during validation (optional) 4671da177e4SLinus Torvalds * @match: called during ematch tree evaluation, must return 1/0 4681da177e4SLinus Torvalds * @destroy: called during destroyage (optional) 4691da177e4SLinus Torvalds * @dump: called during dumping process (optional) 4701da177e4SLinus Torvalds * @owner: owner, must be set to THIS_MODULE 4711da177e4SLinus Torvalds * @link: link to previous/next ematch module (internal use) 4721da177e4SLinus Torvalds */ 473fd2c3ef7SEric Dumazet struct tcf_ematch_ops { 4741da177e4SLinus Torvalds int kind; 4751da177e4SLinus Torvalds int datalen; 47682a470f1SJohn Fastabend int (*change)(struct net *net, void *, 4771da177e4SLinus Torvalds int, struct tcf_ematch *); 4781da177e4SLinus Torvalds int (*match)(struct sk_buff *, struct tcf_ematch *, 4791da177e4SLinus Torvalds struct tcf_pkt_info *); 48082a470f1SJohn Fastabend void (*destroy)(struct tcf_ematch *); 4811da177e4SLinus Torvalds int (*dump)(struct sk_buff *, struct tcf_ematch *); 4821da177e4SLinus Torvalds struct module *owner; 4831da177e4SLinus Torvalds struct list_head link; 4841da177e4SLinus Torvalds }; 4851da177e4SLinus Torvalds 4865c15257fSJoe Perches int tcf_em_register(struct tcf_ematch_ops *); 4875c15257fSJoe Perches void tcf_em_unregister(struct tcf_ematch_ops *); 4885c15257fSJoe Perches int tcf_em_tree_validate(struct tcf_proto *, struct nlattr *, 4891da177e4SLinus Torvalds struct tcf_ematch_tree *); 49082a470f1SJohn Fastabend void tcf_em_tree_destroy(struct tcf_ematch_tree *); 4915c15257fSJoe Perches int tcf_em_tree_dump(struct sk_buff *, struct tcf_ematch_tree *, int); 4925c15257fSJoe Perches int __tcf_em_tree_match(struct sk_buff *, struct tcf_ematch_tree *, 4931da177e4SLinus Torvalds struct tcf_pkt_info *); 4941da177e4SLinus Torvalds 4951da177e4SLinus Torvalds /** 4961da177e4SLinus Torvalds * tcf_em_tree_match - evaulate an ematch tree 4971da177e4SLinus Torvalds * 4981da177e4SLinus Torvalds * @skb: socket buffer of the packet in question 4991da177e4SLinus Torvalds * @tree: ematch tree to be used for evaluation 5001da177e4SLinus Torvalds * @info: packet information examined by classifier 5011da177e4SLinus Torvalds * 5021da177e4SLinus Torvalds * This function matches @skb against the ematch tree in @tree by going 5031da177e4SLinus Torvalds * through all ematches respecting their logic relations returning 5041da177e4SLinus Torvalds * as soon as the result is obvious. 5051da177e4SLinus Torvalds * 5061da177e4SLinus Torvalds * Returns 1 if the ematch tree as-one matches, no ematches are configured 5071da177e4SLinus Torvalds * or ematch is not enabled in the kernel, otherwise 0 is returned. 5081da177e4SLinus Torvalds */ 5091da177e4SLinus Torvalds static inline int tcf_em_tree_match(struct sk_buff *skb, 5101da177e4SLinus Torvalds struct tcf_ematch_tree *tree, 5111da177e4SLinus Torvalds struct tcf_pkt_info *info) 5121da177e4SLinus Torvalds { 5131da177e4SLinus Torvalds if (tree->hdr.nmatches) 5141da177e4SLinus Torvalds return __tcf_em_tree_match(skb, tree, info); 5151da177e4SLinus Torvalds else 5161da177e4SLinus Torvalds return 1; 5171da177e4SLinus Torvalds } 5181da177e4SLinus Torvalds 519db3d99c0SPatrick McHardy #define MODULE_ALIAS_TCF_EMATCH(kind) MODULE_ALIAS("ematch-kind-" __stringify(kind)) 520db3d99c0SPatrick McHardy 5211da177e4SLinus Torvalds #else /* CONFIG_NET_EMATCH */ 5221da177e4SLinus Torvalds 523fd2c3ef7SEric Dumazet struct tcf_ematch_tree { 5241da177e4SLinus Torvalds }; 5251da177e4SLinus Torvalds 5261da177e4SLinus Torvalds #define tcf_em_tree_validate(tp, tb, t) ((void)(t), 0) 52782a470f1SJohn Fastabend #define tcf_em_tree_destroy(t) do { (void)(t); } while(0) 5281da177e4SLinus Torvalds #define tcf_em_tree_dump(skb, t, tlv) (0) 5291da177e4SLinus Torvalds #define tcf_em_tree_match(skb, t, info) ((void)(info), 1) 5301da177e4SLinus Torvalds 5311da177e4SLinus Torvalds #endif /* CONFIG_NET_EMATCH */ 5321da177e4SLinus Torvalds 5331da177e4SLinus Torvalds static inline unsigned char * tcf_get_base_ptr(struct sk_buff *skb, int layer) 5341da177e4SLinus Torvalds { 5351da177e4SLinus Torvalds switch (layer) { 5361da177e4SLinus Torvalds case TCF_LAYER_LINK: 537d3303a65SWolfgang Bumiller return skb_mac_header(skb); 5381da177e4SLinus Torvalds case TCF_LAYER_NETWORK: 539d56f90a7SArnaldo Carvalho de Melo return skb_network_header(skb); 5401da177e4SLinus Torvalds case TCF_LAYER_TRANSPORT: 5419c70220bSArnaldo Carvalho de Melo return skb_transport_header(skb); 5421da177e4SLinus Torvalds } 5431da177e4SLinus Torvalds 5441da177e4SLinus Torvalds return NULL; 5451da177e4SLinus Torvalds } 5461da177e4SLinus Torvalds 547eddc9ec5SArnaldo Carvalho de Melo static inline int tcf_valid_offset(const struct sk_buff *skb, 548eddc9ec5SArnaldo Carvalho de Melo const unsigned char *ptr, const int len) 5491da177e4SLinus Torvalds { 550da521b2cSDavid S. Miller return likely((ptr + len) <= skb_tail_pointer(skb) && 551da521b2cSDavid S. Miller ptr >= skb->head && 552da521b2cSDavid S. Miller (ptr <= (ptr + len))); 5531da177e4SLinus Torvalds } 5541da177e4SLinus Torvalds 5551da177e4SLinus Torvalds static inline int 5561057c55fSAlexander Aring tcf_change_indev(struct net *net, struct nlattr *indev_tlv, 5571057c55fSAlexander Aring struct netlink_ext_ack *extack) 5581da177e4SLinus Torvalds { 5592519a602SWANG Cong char indev[IFNAMSIZ]; 560c01003c2SPatrick McHardy struct net_device *dev; 561c01003c2SPatrick McHardy 562872f6903SFrancis Laniel if (nla_strscpy(indev, indev_tlv, IFNAMSIZ) < 0) { 563e4a58ef3SGuillaume Nault NL_SET_ERR_MSG_ATTR(extack, indev_tlv, 564e4a58ef3SGuillaume Nault "Interface name too long"); 5652519a602SWANG Cong return -EINVAL; 5661057c55fSAlexander Aring } 5672519a602SWANG Cong dev = __dev_get_by_name(net, indev); 568e4a58ef3SGuillaume Nault if (!dev) { 569e4a58ef3SGuillaume Nault NL_SET_ERR_MSG_ATTR(extack, indev_tlv, 570e4a58ef3SGuillaume Nault "Network device not found"); 5712519a602SWANG Cong return -ENODEV; 572e4a58ef3SGuillaume Nault } 5732519a602SWANG Cong return dev->ifindex; 5741da177e4SLinus Torvalds } 5751da177e4SLinus Torvalds 5762519a602SWANG Cong static inline bool 5772519a602SWANG Cong tcf_match_indev(struct sk_buff *skb, int ifindex) 5782519a602SWANG Cong { 5792519a602SWANG Cong if (!ifindex) 5802519a602SWANG Cong return true; 5812519a602SWANG Cong if (!skb->skb_iif) 5822519a602SWANG Cong return false; 5832519a602SWANG Cong return ifindex == skb->skb_iif; 5841da177e4SLinus Torvalds } 5851da177e4SLinus Torvalds 5869c1c0e12SBaowen Zheng int tc_setup_offload_action(struct flow_action *flow_action, 587c2ccf84eSIdo Schimmel const struct tcf_exts *exts, 588c2ccf84eSIdo Schimmel struct netlink_ext_ack *extack); 5899c1c0e12SBaowen Zheng void tc_cleanup_offload_action(struct flow_action *flow_action); 5908cbfe939SBaowen Zheng int tc_setup_action(struct flow_action *flow_action, 591c2ccf84eSIdo Schimmel struct tc_action *actions[], 59280cd22c3SPaul Blakey u32 miss_cookie_base, 593c2ccf84eSIdo Schimmel struct netlink_ext_ack *extack); 5945a6ff4b1SVlad Buslov 595aeb3fecdSCong Wang int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 59640119211SVlad Buslov void *type_data, bool err_stop, bool rtnl_held); 59740119211SVlad Buslov int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp, 59840119211SVlad Buslov enum tc_setup_type type, void *type_data, bool err_stop, 59940119211SVlad Buslov u32 *flags, unsigned int *in_hw_count, bool rtnl_held); 60040119211SVlad Buslov int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp, 60140119211SVlad Buslov enum tc_setup_type type, void *type_data, bool err_stop, 60240119211SVlad Buslov u32 *old_flags, unsigned int *old_in_hw_count, 60340119211SVlad Buslov u32 *new_flags, unsigned int *new_in_hw_count, 60440119211SVlad Buslov bool rtnl_held); 60540119211SVlad Buslov int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp, 60640119211SVlad Buslov enum tc_setup_type type, void *type_data, bool err_stop, 60740119211SVlad Buslov u32 *flags, unsigned int *in_hw_count, bool rtnl_held); 60840119211SVlad Buslov int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp, 60940119211SVlad Buslov bool add, flow_setup_cb_t *cb, 61040119211SVlad Buslov enum tc_setup_type type, void *type_data, 61140119211SVlad Buslov void *cb_priv, u32 *flags, unsigned int *in_hw_count); 612e3ab786bSPablo Neira Ayuso unsigned int tcf_exts_num_actions(struct tcf_exts *exts); 613717503b9SJiri Pirko 6143625750fSPetr Machata #ifdef CONFIG_NET_CLS_ACT 6153625750fSPetr Machata int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch, 6163625750fSPetr Machata enum flow_block_binder_type binder_type, 6173625750fSPetr Machata struct nlattr *block_index_attr, 6183625750fSPetr Machata struct netlink_ext_ack *extack); 6193625750fSPetr Machata void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch); 6203625750fSPetr Machata int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr, 6213625750fSPetr Machata struct netlink_ext_ack *extack); 6223625750fSPetr Machata struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb, 62355f656cdSPetr Machata struct sk_buff **to_free, int *ret); 6243625750fSPetr Machata int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe); 6253625750fSPetr Machata #else 6263625750fSPetr Machata static inline int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch, 6273625750fSPetr Machata enum flow_block_binder_type binder_type, 6283625750fSPetr Machata struct nlattr *block_index_attr, 6293625750fSPetr Machata struct netlink_ext_ack *extack) 6303625750fSPetr Machata { 6313625750fSPetr Machata return 0; 6323625750fSPetr Machata } 6333625750fSPetr Machata 6343625750fSPetr Machata static inline void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch) 6353625750fSPetr Machata { 6363625750fSPetr Machata } 6373625750fSPetr Machata 6383625750fSPetr Machata static inline int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr, 6393625750fSPetr Machata struct netlink_ext_ack *extack) 6403625750fSPetr Machata { 6413625750fSPetr Machata return 0; 6423625750fSPetr Machata } 6433625750fSPetr Machata 6443625750fSPetr Machata static inline struct sk_buff * 6453625750fSPetr Machata tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb, 64655f656cdSPetr Machata struct sk_buff **to_free, int *ret) 6473625750fSPetr Machata { 6483625750fSPetr Machata return skb; 6493625750fSPetr Machata } 6503625750fSPetr Machata 6513625750fSPetr Machata static inline int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe) 6523625750fSPetr Machata { 6533625750fSPetr Machata return 0; 6543625750fSPetr Machata } 6553625750fSPetr Machata #endif 6563625750fSPetr Machata 657a1b7c5fdSJohn Fastabend struct tc_cls_u32_knode { 658a1b7c5fdSJohn Fastabend struct tcf_exts *exts; 659068ceb35SJakub Kicinski struct tcf_result *res; 660e014860eSJohn Fastabend struct tc_u32_sel *sel; 661a1b7c5fdSJohn Fastabend u32 handle; 662a1b7c5fdSJohn Fastabend u32 val; 663a1b7c5fdSJohn Fastabend u32 mask; 664a1b7c5fdSJohn Fastabend u32 link_handle; 665e014860eSJohn Fastabend u8 fshift; 666a1b7c5fdSJohn Fastabend }; 667a1b7c5fdSJohn Fastabend 668a1b7c5fdSJohn Fastabend struct tc_cls_u32_hnode { 669a1b7c5fdSJohn Fastabend u32 handle; 670a1b7c5fdSJohn Fastabend u32 prio; 671a1b7c5fdSJohn Fastabend unsigned int divisor; 672a1b7c5fdSJohn Fastabend }; 673a1b7c5fdSJohn Fastabend 674a1b7c5fdSJohn Fastabend enum tc_clsu32_command { 675a1b7c5fdSJohn Fastabend TC_CLSU32_NEW_KNODE, 676a1b7c5fdSJohn Fastabend TC_CLSU32_REPLACE_KNODE, 677a1b7c5fdSJohn Fastabend TC_CLSU32_DELETE_KNODE, 678a1b7c5fdSJohn Fastabend TC_CLSU32_NEW_HNODE, 679a1b7c5fdSJohn Fastabend TC_CLSU32_REPLACE_HNODE, 680a1b7c5fdSJohn Fastabend TC_CLSU32_DELETE_HNODE, 681a1b7c5fdSJohn Fastabend }; 682a1b7c5fdSJohn Fastabend 683a1b7c5fdSJohn Fastabend struct tc_cls_u32_offload { 684f9e30088SPablo Neira Ayuso struct flow_cls_common_offload common; 685a1b7c5fdSJohn Fastabend /* knode values */ 686a1b7c5fdSJohn Fastabend enum tc_clsu32_command command; 687a1b7c5fdSJohn Fastabend union { 688a1b7c5fdSJohn Fastabend struct tc_cls_u32_knode knode; 689a1b7c5fdSJohn Fastabend struct tc_cls_u32_hnode hnode; 690a1b7c5fdSJohn Fastabend }; 691a1b7c5fdSJohn Fastabend }; 692a1b7c5fdSJohn Fastabend 6937b06e8aeSJiri Pirko static inline bool tc_can_offload(const struct net_device *dev) 6946843e7a2SJohn Fastabend { 69570b5aee4SJiri Pirko return dev->features & NETIF_F_HW_TC; 6966843e7a2SJohn Fastabend } 6976843e7a2SJohn Fastabend 698f9eda14fSQuentin Monnet static inline bool tc_can_offload_extack(const struct net_device *dev, 699f9eda14fSQuentin Monnet struct netlink_ext_ack *extack) 700f9eda14fSQuentin Monnet { 701f9eda14fSQuentin Monnet bool can = tc_can_offload(dev); 702f9eda14fSQuentin Monnet 703f9eda14fSQuentin Monnet if (!can) 704f9eda14fSQuentin Monnet NL_SET_ERR_MSG(extack, "TC offload is disabled on net device"); 705f9eda14fSQuentin Monnet 706f9eda14fSQuentin Monnet return can; 707f9eda14fSQuentin Monnet } 708f9eda14fSQuentin Monnet 709878db9f0SJakub Kicinski static inline bool 710878db9f0SJakub Kicinski tc_cls_can_offload_and_chain0(const struct net_device *dev, 711f9e30088SPablo Neira Ayuso struct flow_cls_common_offload *common) 712878db9f0SJakub Kicinski { 713878db9f0SJakub Kicinski if (!tc_can_offload_extack(dev, common->extack)) 714878db9f0SJakub Kicinski return false; 715878db9f0SJakub Kicinski if (common->chain_index) { 716878db9f0SJakub Kicinski NL_SET_ERR_MSG(common->extack, 717878db9f0SJakub Kicinski "Driver supports only offload of chain 0"); 718878db9f0SJakub Kicinski return false; 719878db9f0SJakub Kicinski } 720878db9f0SJakub Kicinski return true; 721878db9f0SJakub Kicinski } 722878db9f0SJakub Kicinski 72355330f05SHadar Hen Zion static inline bool tc_skip_hw(u32 flags) 72455330f05SHadar Hen Zion { 72555330f05SHadar Hen Zion return (flags & TCA_CLS_FLAGS_SKIP_HW) ? true : false; 72655330f05SHadar Hen Zion } 72755330f05SHadar Hen Zion 728d34e3e18SSamudrala, Sridhar static inline bool tc_skip_sw(u32 flags) 729d34e3e18SSamudrala, Sridhar { 730d34e3e18SSamudrala, Sridhar return (flags & TCA_CLS_FLAGS_SKIP_SW) ? true : false; 731d34e3e18SSamudrala, Sridhar } 732d34e3e18SSamudrala, Sridhar 733d34e3e18SSamudrala, Sridhar /* SKIP_HW and SKIP_SW are mutually exclusive flags. */ 734d34e3e18SSamudrala, Sridhar static inline bool tc_flags_valid(u32 flags) 735d34e3e18SSamudrala, Sridhar { 73681c7288bSMarcelo Ricardo Leitner if (flags & ~(TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW | 73781c7288bSMarcelo Ricardo Leitner TCA_CLS_FLAGS_VERBOSE)) 738d34e3e18SSamudrala, Sridhar return false; 739d34e3e18SSamudrala, Sridhar 74081c7288bSMarcelo Ricardo Leitner flags &= TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW; 741d34e3e18SSamudrala, Sridhar if (!(flags ^ (TCA_CLS_FLAGS_SKIP_HW | TCA_CLS_FLAGS_SKIP_SW))) 742d34e3e18SSamudrala, Sridhar return false; 743d34e3e18SSamudrala, Sridhar 744d34e3e18SSamudrala, Sridhar return true; 745d34e3e18SSamudrala, Sridhar } 746d34e3e18SSamudrala, Sridhar 747e696028aSOr Gerlitz static inline bool tc_in_hw(u32 flags) 748e696028aSOr Gerlitz { 749e696028aSOr Gerlitz return (flags & TCA_CLS_FLAGS_IN_HW) ? true : false; 750e696028aSOr Gerlitz } 751e696028aSOr Gerlitz 75234832e1cSJakub Kicinski static inline void 753f9e30088SPablo Neira Ayuso tc_cls_common_offload_init(struct flow_cls_common_offload *cls_common, 75434832e1cSJakub Kicinski const struct tcf_proto *tp, u32 flags, 75534832e1cSJakub Kicinski struct netlink_ext_ack *extack) 75634832e1cSJakub Kicinski { 75734832e1cSJakub Kicinski cls_common->chain_index = tp->chain->index; 75834832e1cSJakub Kicinski cls_common->protocol = tp->protocol; 759ef01adaeSPablo Neira Ayuso cls_common->prio = tp->prio >> 16; 76081c7288bSMarcelo Ricardo Leitner if (tc_skip_sw(flags) || flags & TCA_CLS_FLAGS_VERBOSE) 76134832e1cSJakub Kicinski cls_common->extack = extack; 76234832e1cSJakub Kicinski } 76334832e1cSJakub Kicinski 7649453d45eSVlad Buslov #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 7659453d45eSVlad Buslov static inline struct tc_skb_ext *tc_skb_ext_alloc(struct sk_buff *skb) 7669453d45eSVlad Buslov { 7679453d45eSVlad Buslov struct tc_skb_ext *tc_skb_ext = skb_ext_add(skb, TC_SKB_EXT); 7689453d45eSVlad Buslov 7699453d45eSVlad Buslov if (tc_skb_ext) 7709453d45eSVlad Buslov memset(tc_skb_ext, 0, sizeof(*tc_skb_ext)); 7719453d45eSVlad Buslov return tc_skb_ext; 7729453d45eSVlad Buslov } 7739453d45eSVlad Buslov #endif 7749453d45eSVlad Buslov 775b87f7936SYotam Gigi enum tc_matchall_command { 776b87f7936SYotam Gigi TC_CLSMATCHALL_REPLACE, 777b87f7936SYotam Gigi TC_CLSMATCHALL_DESTROY, 778b7fe4ab8SPieter Jansen van Vuuren TC_CLSMATCHALL_STATS, 779b87f7936SYotam Gigi }; 780b87f7936SYotam Gigi 781b87f7936SYotam Gigi struct tc_cls_matchall_offload { 782f9e30088SPablo Neira Ayuso struct flow_cls_common_offload common; 783b87f7936SYotam Gigi enum tc_matchall_command command; 784f00cbf19SPieter Jansen van Vuuren struct flow_rule *rule; 785b7fe4ab8SPieter Jansen van Vuuren struct flow_stats stats; 7865246c896SOz Shlomo bool use_act_stats; 787b87f7936SYotam Gigi unsigned long cookie; 788b87f7936SYotam Gigi }; 789b87f7936SYotam Gigi 790332ae8e2SJakub Kicinski enum tc_clsbpf_command { 791102740bdSJakub Kicinski TC_CLSBPF_OFFLOAD, 79268d64063SJakub Kicinski TC_CLSBPF_STATS, 793332ae8e2SJakub Kicinski }; 794332ae8e2SJakub Kicinski 795332ae8e2SJakub Kicinski struct tc_cls_bpf_offload { 796f9e30088SPablo Neira Ayuso struct flow_cls_common_offload common; 797332ae8e2SJakub Kicinski enum tc_clsbpf_command command; 798332ae8e2SJakub Kicinski struct tcf_exts *exts; 799332ae8e2SJakub Kicinski struct bpf_prog *prog; 800102740bdSJakub Kicinski struct bpf_prog *oldprog; 801332ae8e2SJakub Kicinski const char *name; 802332ae8e2SJakub Kicinski bool exts_integrated; 803332ae8e2SJakub Kicinski }; 804332ae8e2SJakub Kicinski 8051045ba77SJamal Hadi Salim /* This structure holds cookie structure that is passed from user 8061045ba77SJamal Hadi Salim * to the kernel for actions and classifiers 8071045ba77SJamal Hadi Salim */ 8081045ba77SJamal Hadi Salim struct tc_cookie { 8091045ba77SJamal Hadi Salim u8 *data; 8101045ba77SJamal Hadi Salim u32 len; 811eec94fdbSVlad Buslov struct rcu_head rcu; 8121045ba77SJamal Hadi Salim }; 813602f3bafSNogah Frankel 814f34b4aacSNogah Frankel struct tc_qopt_offload_stats { 81550dc9a85SAhmed S. Darwish struct gnet_stats_basic_sync *bstats; 816f34b4aacSNogah Frankel struct gnet_stats_queue *qstats; 817f34b4aacSNogah Frankel }; 818f34b4aacSNogah Frankel 819f971b132SJakub Kicinski enum tc_mq_command { 820f971b132SJakub Kicinski TC_MQ_CREATE, 821f971b132SJakub Kicinski TC_MQ_DESTROY, 82247c669a4SJakub Kicinski TC_MQ_STATS, 823d577a3d2SJakub Kicinski TC_MQ_GRAFT, 824d577a3d2SJakub Kicinski }; 825d577a3d2SJakub Kicinski 826d577a3d2SJakub Kicinski struct tc_mq_opt_offload_graft_params { 827d577a3d2SJakub Kicinski unsigned long queue; 828d577a3d2SJakub Kicinski u32 child_handle; 829f971b132SJakub Kicinski }; 830f971b132SJakub Kicinski 831f971b132SJakub Kicinski struct tc_mq_qopt_offload { 832f971b132SJakub Kicinski enum tc_mq_command command; 833f971b132SJakub Kicinski u32 handle; 834d577a3d2SJakub Kicinski union { 83547c669a4SJakub Kicinski struct tc_qopt_offload_stats stats; 836d577a3d2SJakub Kicinski struct tc_mq_opt_offload_graft_params graft_params; 837d577a3d2SJakub Kicinski }; 838f971b132SJakub Kicinski }; 839f971b132SJakub Kicinski 840d03b195bSMaxim Mikityanskiy enum tc_htb_command { 841d03b195bSMaxim Mikityanskiy /* Root */ 842d03b195bSMaxim Mikityanskiy TC_HTB_CREATE, /* Initialize HTB offload. */ 843d03b195bSMaxim Mikityanskiy TC_HTB_DESTROY, /* Destroy HTB offload. */ 844d03b195bSMaxim Mikityanskiy 845d03b195bSMaxim Mikityanskiy /* Classes */ 846d03b195bSMaxim Mikityanskiy /* Allocate qid and create leaf. */ 847d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_ALLOC_QUEUE, 848d03b195bSMaxim Mikityanskiy /* Convert leaf to inner, preserve and return qid, create new leaf. */ 849d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_TO_INNER, 850d03b195bSMaxim Mikityanskiy /* Delete leaf, while siblings remain. */ 851d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_DEL, 852d03b195bSMaxim Mikityanskiy /* Delete leaf, convert parent to leaf, preserving qid. */ 853d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_DEL_LAST, 854d03b195bSMaxim Mikityanskiy /* TC_HTB_LEAF_DEL_LAST, but delete driver data on hardware errors. */ 855d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_DEL_LAST_FORCE, 856d03b195bSMaxim Mikityanskiy /* Modify parameters of a node. */ 857d03b195bSMaxim Mikityanskiy TC_HTB_NODE_MODIFY, 858d03b195bSMaxim Mikityanskiy 859d03b195bSMaxim Mikityanskiy /* Class qdisc */ 860d03b195bSMaxim Mikityanskiy TC_HTB_LEAF_QUERY_QUEUE, /* Query qid by classid. */ 861d03b195bSMaxim Mikityanskiy }; 862d03b195bSMaxim Mikityanskiy 863d03b195bSMaxim Mikityanskiy struct tc_htb_qopt_offload { 864d03b195bSMaxim Mikityanskiy struct netlink_ext_ack *extack; 865d03b195bSMaxim Mikityanskiy enum tc_htb_command command; 866d03b195bSMaxim Mikityanskiy u32 parent_classid; 867ca49bfd9SMaxim Mikityanskiy u16 classid; 868d03b195bSMaxim Mikityanskiy u16 qid; 869d03b195bSMaxim Mikityanskiy u64 rate; 870d03b195bSMaxim Mikityanskiy u64 ceil; 871*12e7789aSNaveen Mamindlapalli u8 prio; 872d03b195bSMaxim Mikityanskiy }; 873d03b195bSMaxim Mikityanskiy 874d03b195bSMaxim Mikityanskiy #define TC_HTB_CLASSID_ROOT U32_MAX 875d03b195bSMaxim Mikityanskiy 876602f3bafSNogah Frankel enum tc_red_command { 877602f3bafSNogah Frankel TC_RED_REPLACE, 878602f3bafSNogah Frankel TC_RED_DESTROY, 879602f3bafSNogah Frankel TC_RED_STATS, 880602f3bafSNogah Frankel TC_RED_XSTATS, 881bf2a752bSJakub Kicinski TC_RED_GRAFT, 882602f3bafSNogah Frankel }; 883602f3bafSNogah Frankel 884602f3bafSNogah Frankel struct tc_red_qopt_offload_params { 885602f3bafSNogah Frankel u32 min; 886602f3bafSNogah Frankel u32 max; 887602f3bafSNogah Frankel u32 probability; 888c0b7490bSJakub Kicinski u32 limit; 889602f3bafSNogah Frankel bool is_ecn; 890190852a5SJakub Kicinski bool is_harddrop; 8910a7fad23SPetr Machata bool is_nodrop; 892416ef9b1SJakub Kicinski struct gnet_stats_queue *qstats; 893602f3bafSNogah Frankel }; 894602f3bafSNogah Frankel 895602f3bafSNogah Frankel struct tc_red_qopt_offload { 896602f3bafSNogah Frankel enum tc_red_command command; 897602f3bafSNogah Frankel u32 handle; 898602f3bafSNogah Frankel u32 parent; 899602f3bafSNogah Frankel union { 900602f3bafSNogah Frankel struct tc_red_qopt_offload_params set; 901f34b4aacSNogah Frankel struct tc_qopt_offload_stats stats; 902602f3bafSNogah Frankel struct red_stats *xstats; 903bf2a752bSJakub Kicinski u32 child_handle; 904602f3bafSNogah Frankel }; 905602f3bafSNogah Frankel }; 906602f3bafSNogah Frankel 907890d8d23SJakub Kicinski enum tc_gred_command { 908890d8d23SJakub Kicinski TC_GRED_REPLACE, 909890d8d23SJakub Kicinski TC_GRED_DESTROY, 910e49efd52SJakub Kicinski TC_GRED_STATS, 911890d8d23SJakub Kicinski }; 912890d8d23SJakub Kicinski 913890d8d23SJakub Kicinski struct tc_gred_vq_qopt_offload_params { 914890d8d23SJakub Kicinski bool present; 915890d8d23SJakub Kicinski u32 limit; 916890d8d23SJakub Kicinski u32 prio; 917890d8d23SJakub Kicinski u32 min; 918890d8d23SJakub Kicinski u32 max; 919890d8d23SJakub Kicinski bool is_ecn; 920890d8d23SJakub Kicinski bool is_harddrop; 921890d8d23SJakub Kicinski u32 probability; 922890d8d23SJakub Kicinski /* Only need backlog, see struct tc_prio_qopt_offload_params */ 923890d8d23SJakub Kicinski u32 *backlog; 924890d8d23SJakub Kicinski }; 925890d8d23SJakub Kicinski 926890d8d23SJakub Kicinski struct tc_gred_qopt_offload_params { 927890d8d23SJakub Kicinski bool grio_on; 928890d8d23SJakub Kicinski bool wred_on; 929890d8d23SJakub Kicinski unsigned int dp_cnt; 930890d8d23SJakub Kicinski unsigned int dp_def; 931890d8d23SJakub Kicinski struct gnet_stats_queue *qstats; 932890d8d23SJakub Kicinski struct tc_gred_vq_qopt_offload_params tab[MAX_DPs]; 933890d8d23SJakub Kicinski }; 934890d8d23SJakub Kicinski 935e49efd52SJakub Kicinski struct tc_gred_qopt_offload_stats { 93650dc9a85SAhmed S. Darwish struct gnet_stats_basic_sync bstats[MAX_DPs]; 937e49efd52SJakub Kicinski struct gnet_stats_queue qstats[MAX_DPs]; 938e49efd52SJakub Kicinski struct red_stats *xstats[MAX_DPs]; 939e49efd52SJakub Kicinski }; 940e49efd52SJakub Kicinski 941890d8d23SJakub Kicinski struct tc_gred_qopt_offload { 942890d8d23SJakub Kicinski enum tc_gred_command command; 943890d8d23SJakub Kicinski u32 handle; 944890d8d23SJakub Kicinski u32 parent; 945890d8d23SJakub Kicinski union { 946890d8d23SJakub Kicinski struct tc_gred_qopt_offload_params set; 947e49efd52SJakub Kicinski struct tc_gred_qopt_offload_stats stats; 948890d8d23SJakub Kicinski }; 949890d8d23SJakub Kicinski }; 950890d8d23SJakub Kicinski 9517fdb61b4SNogah Frankel enum tc_prio_command { 9527fdb61b4SNogah Frankel TC_PRIO_REPLACE, 9537fdb61b4SNogah Frankel TC_PRIO_DESTROY, 9547fdb61b4SNogah Frankel TC_PRIO_STATS, 955b9c7a7acSNogah Frankel TC_PRIO_GRAFT, 9567fdb61b4SNogah Frankel }; 9577fdb61b4SNogah Frankel 9587fdb61b4SNogah Frankel struct tc_prio_qopt_offload_params { 9597fdb61b4SNogah Frankel int bands; 9607fdb61b4SNogah Frankel u8 priomap[TC_PRIO_MAX + 1]; 9619586a992SPetr Machata /* At the point of un-offloading the Qdisc, the reported backlog and 9629586a992SPetr Machata * qlen need to be reduced by the portion that is in HW. 9637fdb61b4SNogah Frankel */ 9647fdb61b4SNogah Frankel struct gnet_stats_queue *qstats; 9657fdb61b4SNogah Frankel }; 9667fdb61b4SNogah Frankel 967b9c7a7acSNogah Frankel struct tc_prio_qopt_offload_graft_params { 968b9c7a7acSNogah Frankel u8 band; 969b9c7a7acSNogah Frankel u32 child_handle; 970b9c7a7acSNogah Frankel }; 971b9c7a7acSNogah Frankel 9727fdb61b4SNogah Frankel struct tc_prio_qopt_offload { 9737fdb61b4SNogah Frankel enum tc_prio_command command; 9747fdb61b4SNogah Frankel u32 handle; 9757fdb61b4SNogah Frankel u32 parent; 9767fdb61b4SNogah Frankel union { 9777fdb61b4SNogah Frankel struct tc_prio_qopt_offload_params replace_params; 9787fdb61b4SNogah Frankel struct tc_qopt_offload_stats stats; 979b9c7a7acSNogah Frankel struct tc_prio_qopt_offload_graft_params graft_params; 9807fdb61b4SNogah Frankel }; 9817fdb61b4SNogah Frankel }; 982b9c7a7acSNogah Frankel 98398b0e5f6SJakub Kicinski enum tc_root_command { 98498b0e5f6SJakub Kicinski TC_ROOT_GRAFT, 98598b0e5f6SJakub Kicinski }; 98698b0e5f6SJakub Kicinski 98798b0e5f6SJakub Kicinski struct tc_root_qopt_offload { 98898b0e5f6SJakub Kicinski enum tc_root_command command; 98998b0e5f6SJakub Kicinski u32 handle; 99098b0e5f6SJakub Kicinski bool ingress; 99198b0e5f6SJakub Kicinski }; 99298b0e5f6SJakub Kicinski 993d35eb52bSPetr Machata enum tc_ets_command { 994d35eb52bSPetr Machata TC_ETS_REPLACE, 995d35eb52bSPetr Machata TC_ETS_DESTROY, 996d35eb52bSPetr Machata TC_ETS_STATS, 997d35eb52bSPetr Machata TC_ETS_GRAFT, 998d35eb52bSPetr Machata }; 999d35eb52bSPetr Machata 1000d35eb52bSPetr Machata struct tc_ets_qopt_offload_replace_params { 1001d35eb52bSPetr Machata unsigned int bands; 1002d35eb52bSPetr Machata u8 priomap[TC_PRIO_MAX + 1]; 1003d35eb52bSPetr Machata unsigned int quanta[TCQ_ETS_MAX_BANDS]; /* 0 for strict bands. */ 1004d35eb52bSPetr Machata unsigned int weights[TCQ_ETS_MAX_BANDS]; 1005d35eb52bSPetr Machata struct gnet_stats_queue *qstats; 1006d35eb52bSPetr Machata }; 1007d35eb52bSPetr Machata 1008d35eb52bSPetr Machata struct tc_ets_qopt_offload_graft_params { 1009d35eb52bSPetr Machata u8 band; 1010d35eb52bSPetr Machata u32 child_handle; 1011d35eb52bSPetr Machata }; 1012d35eb52bSPetr Machata 1013d35eb52bSPetr Machata struct tc_ets_qopt_offload { 1014d35eb52bSPetr Machata enum tc_ets_command command; 1015d35eb52bSPetr Machata u32 handle; 1016d35eb52bSPetr Machata u32 parent; 1017d35eb52bSPetr Machata union { 1018d35eb52bSPetr Machata struct tc_ets_qopt_offload_replace_params replace_params; 1019d35eb52bSPetr Machata struct tc_qopt_offload_stats stats; 1020d35eb52bSPetr Machata struct tc_ets_qopt_offload_graft_params graft_params; 1021d35eb52bSPetr Machata }; 1022d35eb52bSPetr Machata }; 1023d35eb52bSPetr Machata 1024ef6aadccSPetr Machata enum tc_tbf_command { 1025ef6aadccSPetr Machata TC_TBF_REPLACE, 1026ef6aadccSPetr Machata TC_TBF_DESTROY, 1027ef6aadccSPetr Machata TC_TBF_STATS, 10286b3efbfaSPetr Machata TC_TBF_GRAFT, 1029ef6aadccSPetr Machata }; 1030ef6aadccSPetr Machata 1031ef6aadccSPetr Machata struct tc_tbf_qopt_offload_replace_params { 1032ef6aadccSPetr Machata struct psched_ratecfg rate; 1033ef6aadccSPetr Machata u32 max_size; 1034ef6aadccSPetr Machata struct gnet_stats_queue *qstats; 1035ef6aadccSPetr Machata }; 1036ef6aadccSPetr Machata 1037ef6aadccSPetr Machata struct tc_tbf_qopt_offload { 1038ef6aadccSPetr Machata enum tc_tbf_command command; 1039ef6aadccSPetr Machata u32 handle; 1040ef6aadccSPetr Machata u32 parent; 1041ef6aadccSPetr Machata union { 1042ef6aadccSPetr Machata struct tc_tbf_qopt_offload_replace_params replace_params; 1043ef6aadccSPetr Machata struct tc_qopt_offload_stats stats; 10446b3efbfaSPetr Machata u32 child_handle; 1045ef6aadccSPetr Machata }; 1046ef6aadccSPetr Machata }; 1047ef6aadccSPetr Machata 1048aaca9408SPetr Machata enum tc_fifo_command { 1049aaca9408SPetr Machata TC_FIFO_REPLACE, 1050aaca9408SPetr Machata TC_FIFO_DESTROY, 1051aaca9408SPetr Machata TC_FIFO_STATS, 1052aaca9408SPetr Machata }; 1053aaca9408SPetr Machata 1054aaca9408SPetr Machata struct tc_fifo_qopt_offload { 1055aaca9408SPetr Machata enum tc_fifo_command command; 1056aaca9408SPetr Machata u32 handle; 1057aaca9408SPetr Machata u32 parent; 1058aaca9408SPetr Machata union { 1059aaca9408SPetr Machata struct tc_qopt_offload_stats stats; 1060aaca9408SPetr Machata }; 1061aaca9408SPetr Machata }; 1062aaca9408SPetr Machata 106335d39fecSPaul Blakey #ifdef CONFIG_NET_CLS_ACT 106435d39fecSPaul Blakey DECLARE_STATIC_KEY_FALSE(tc_skb_ext_tc); 106535d39fecSPaul Blakey void tc_skb_ext_tc_enable(void); 106635d39fecSPaul Blakey void tc_skb_ext_tc_disable(void); 106735d39fecSPaul Blakey #define tc_skb_ext_tc_enabled() static_branch_unlikely(&tc_skb_ext_tc) 106835d39fecSPaul Blakey #else /* CONFIG_NET_CLS_ACT */ 106935d39fecSPaul Blakey static inline void tc_skb_ext_tc_enable(void) { } 107035d39fecSPaul Blakey static inline void tc_skb_ext_tc_disable(void) { } 107135d39fecSPaul Blakey #define tc_skb_ext_tc_enabled() false 107235d39fecSPaul Blakey #endif 107335d39fecSPaul Blakey 10741da177e4SLinus Torvalds #endif 1075