12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * net/sched/cls_route.c ROUTE4 classifier.
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
61da177e4SLinus Torvalds */
71da177e4SLinus Torvalds
81da177e4SLinus Torvalds #include <linux/module.h>
95a0e3ad6STejun Heo #include <linux/slab.h>
101da177e4SLinus Torvalds #include <linux/types.h>
111da177e4SLinus Torvalds #include <linux/kernel.h>
121da177e4SLinus Torvalds #include <linux/string.h>
131da177e4SLinus Torvalds #include <linux/errno.h>
141da177e4SLinus Torvalds #include <linux/skbuff.h>
150ba48053SPatrick McHardy #include <net/dst.h>
160ba48053SPatrick McHardy #include <net/route.h>
170ba48053SPatrick McHardy #include <net/netlink.h>
181da177e4SLinus Torvalds #include <net/act_api.h>
191da177e4SLinus Torvalds #include <net/pkt_cls.h>
20*9f3101dcSPedro Tammela #include <net/tc_wrapper.h>
211da177e4SLinus Torvalds
221da177e4SLinus Torvalds /*
23cc7ec456SEric Dumazet * 1. For now we assume that route tags < 256.
24cc7ec456SEric Dumazet * It allows to use direct table lookups, instead of hash tables.
25cc7ec456SEric Dumazet * 2. For now we assume that "from TAG" and "fromdev DEV" statements
26cc7ec456SEric Dumazet * are mutually exclusive.
27cc7ec456SEric Dumazet * 3. "to TAG from ANY" has higher priority, than "to ANY from XXX"
281da177e4SLinus Torvalds */
29cc7ec456SEric Dumazet struct route4_fastmap {
301da177e4SLinus Torvalds struct route4_filter *filter;
311da177e4SLinus Torvalds u32 id;
321da177e4SLinus Torvalds int iif;
331da177e4SLinus Torvalds };
341da177e4SLinus Torvalds
35cc7ec456SEric Dumazet struct route4_head {
361da177e4SLinus Torvalds struct route4_fastmap fastmap[16];
371109c005SJohn Fastabend struct route4_bucket __rcu *table[256 + 1];
381109c005SJohn Fastabend struct rcu_head rcu;
391da177e4SLinus Torvalds };
401da177e4SLinus Torvalds
41cc7ec456SEric Dumazet struct route4_bucket {
421da177e4SLinus Torvalds /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */
431109c005SJohn Fastabend struct route4_filter __rcu *ht[16 + 16 + 1];
441109c005SJohn Fastabend struct rcu_head rcu;
451da177e4SLinus Torvalds };
461da177e4SLinus Torvalds
47cc7ec456SEric Dumazet struct route4_filter {
481109c005SJohn Fastabend struct route4_filter __rcu *next;
491da177e4SLinus Torvalds u32 id;
501da177e4SLinus Torvalds int iif;
511da177e4SLinus Torvalds
521da177e4SLinus Torvalds struct tcf_result res;
531da177e4SLinus Torvalds struct tcf_exts exts;
541da177e4SLinus Torvalds u32 handle;
551da177e4SLinus Torvalds struct route4_bucket *bkt;
561109c005SJohn Fastabend struct tcf_proto *tp;
57aaa908ffSCong Wang struct rcu_work rwork;
58c2f3f31dSCong Wang };
591da177e4SLinus Torvalds
601da177e4SLinus Torvalds #define ROUTE4_FAILURE ((struct route4_filter *)(-1L))
611da177e4SLinus Torvalds
route4_fastmap_hash(u32 id,int iif)62cc7ec456SEric Dumazet static inline int route4_fastmap_hash(u32 id, int iif)
631da177e4SLinus Torvalds {
641da177e4SLinus Torvalds return id & 0xF;
651da177e4SLinus Torvalds }
661da177e4SLinus Torvalds
671109c005SJohn Fastabend static DEFINE_SPINLOCK(fastmap_lock);
68cc7ec456SEric Dumazet static void
route4_reset_fastmap(struct route4_head * head)691109c005SJohn Fastabend route4_reset_fastmap(struct route4_head *head)
701da177e4SLinus Torvalds {
711109c005SJohn Fastabend spin_lock_bh(&fastmap_lock);
721da177e4SLinus Torvalds memset(head->fastmap, 0, sizeof(head->fastmap));
731109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock);
741da177e4SLinus Torvalds }
751da177e4SLinus Torvalds
76cc7ec456SEric Dumazet static void
route4_set_fastmap(struct route4_head * head,u32 id,int iif,struct route4_filter * f)771da177e4SLinus Torvalds route4_set_fastmap(struct route4_head *head, u32 id, int iif,
781da177e4SLinus Torvalds struct route4_filter *f)
791da177e4SLinus Torvalds {
801da177e4SLinus Torvalds int h = route4_fastmap_hash(id, iif);
81cc7ec456SEric Dumazet
821109c005SJohn Fastabend /* fastmap updates must look atomic to aling id, iff, filter */
831109c005SJohn Fastabend spin_lock_bh(&fastmap_lock);
841da177e4SLinus Torvalds head->fastmap[h].id = id;
851da177e4SLinus Torvalds head->fastmap[h].iif = iif;
861da177e4SLinus Torvalds head->fastmap[h].filter = f;
871109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock);
881da177e4SLinus Torvalds }
891da177e4SLinus Torvalds
route4_hash_to(u32 id)90cc7ec456SEric Dumazet static inline int route4_hash_to(u32 id)
911da177e4SLinus Torvalds {
921da177e4SLinus Torvalds return id & 0xFF;
931da177e4SLinus Torvalds }
941da177e4SLinus Torvalds
route4_hash_from(u32 id)95cc7ec456SEric Dumazet static inline int route4_hash_from(u32 id)
961da177e4SLinus Torvalds {
971da177e4SLinus Torvalds return (id >> 16) & 0xF;
981da177e4SLinus Torvalds }
991da177e4SLinus Torvalds
route4_hash_iif(int iif)100cc7ec456SEric Dumazet static inline int route4_hash_iif(int iif)
1011da177e4SLinus Torvalds {
1021da177e4SLinus Torvalds return 16 + ((iif >> 16) & 0xF);
1031da177e4SLinus Torvalds }
1041da177e4SLinus Torvalds
route4_hash_wild(void)105cc7ec456SEric Dumazet static inline int route4_hash_wild(void)
1061da177e4SLinus Torvalds {
1071da177e4SLinus Torvalds return 32;
1081da177e4SLinus Torvalds }
1091da177e4SLinus Torvalds
1101da177e4SLinus Torvalds #define ROUTE4_APPLY_RESULT() \
1111da177e4SLinus Torvalds { \
1121da177e4SLinus Torvalds *res = f->res; \
1136fc6d06eSJiri Pirko if (tcf_exts_has_actions(&f->exts)) { \
1141da177e4SLinus Torvalds int r = tcf_exts_exec(skb, &f->exts, res); \
1151da177e4SLinus Torvalds if (r < 0) { \
1161da177e4SLinus Torvalds dont_cache = 1; \
1171da177e4SLinus Torvalds continue; \
1181da177e4SLinus Torvalds } \
1191da177e4SLinus Torvalds return r; \
1201da177e4SLinus Torvalds } else if (!dont_cache) \
1211da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, f); \
1221da177e4SLinus Torvalds return 0; \
1231da177e4SLinus Torvalds }
1241da177e4SLinus Torvalds
route4_classify(struct sk_buff * skb,const struct tcf_proto * tp,struct tcf_result * res)125*9f3101dcSPedro Tammela TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb,
126*9f3101dcSPedro Tammela const struct tcf_proto *tp,
1271da177e4SLinus Torvalds struct tcf_result *res)
1281da177e4SLinus Torvalds {
1291109c005SJohn Fastabend struct route4_head *head = rcu_dereference_bh(tp->root);
1301da177e4SLinus Torvalds struct dst_entry *dst;
1311da177e4SLinus Torvalds struct route4_bucket *b;
1321da177e4SLinus Torvalds struct route4_filter *f;
1331da177e4SLinus Torvalds u32 id, h;
1341da177e4SLinus Torvalds int iif, dont_cache = 0;
1351da177e4SLinus Torvalds
136cc7ec456SEric Dumazet dst = skb_dst(skb);
137cc7ec456SEric Dumazet if (!dst)
1381da177e4SLinus Torvalds goto failure;
1391da177e4SLinus Torvalds
1401da177e4SLinus Torvalds id = dst->tclassid;
1411da177e4SLinus Torvalds
14292101b3bSDavid S. Miller iif = inet_iif(skb);
1431da177e4SLinus Torvalds
1441da177e4SLinus Torvalds h = route4_fastmap_hash(id, iif);
1451109c005SJohn Fastabend
1461109c005SJohn Fastabend spin_lock(&fastmap_lock);
1471da177e4SLinus Torvalds if (id == head->fastmap[h].id &&
1481da177e4SLinus Torvalds iif == head->fastmap[h].iif &&
1491da177e4SLinus Torvalds (f = head->fastmap[h].filter) != NULL) {
1501109c005SJohn Fastabend if (f == ROUTE4_FAILURE) {
1511109c005SJohn Fastabend spin_unlock(&fastmap_lock);
1521da177e4SLinus Torvalds goto failure;
1531109c005SJohn Fastabend }
1541da177e4SLinus Torvalds
1551da177e4SLinus Torvalds *res = f->res;
1561109c005SJohn Fastabend spin_unlock(&fastmap_lock);
1571da177e4SLinus Torvalds return 0;
1581da177e4SLinus Torvalds }
1591109c005SJohn Fastabend spin_unlock(&fastmap_lock);
1601da177e4SLinus Torvalds
1611da177e4SLinus Torvalds h = route4_hash_to(id);
1621da177e4SLinus Torvalds
1631da177e4SLinus Torvalds restart:
1641109c005SJohn Fastabend b = rcu_dereference_bh(head->table[h]);
165cc7ec456SEric Dumazet if (b) {
1661109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]);
1671109c005SJohn Fastabend f;
1681109c005SJohn Fastabend f = rcu_dereference_bh(f->next))
1691da177e4SLinus Torvalds if (f->id == id)
1701da177e4SLinus Torvalds ROUTE4_APPLY_RESULT();
1711da177e4SLinus Torvalds
1721109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]);
1731109c005SJohn Fastabend f;
1741109c005SJohn Fastabend f = rcu_dereference_bh(f->next))
1751da177e4SLinus Torvalds if (f->iif == iif)
1761da177e4SLinus Torvalds ROUTE4_APPLY_RESULT();
1771da177e4SLinus Torvalds
1781109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]);
1791109c005SJohn Fastabend f;
1801109c005SJohn Fastabend f = rcu_dereference_bh(f->next))
1811da177e4SLinus Torvalds ROUTE4_APPLY_RESULT();
1821da177e4SLinus Torvalds }
1831da177e4SLinus Torvalds if (h < 256) {
1841da177e4SLinus Torvalds h = 256;
1851da177e4SLinus Torvalds id &= ~0xFFFF;
1861da177e4SLinus Torvalds goto restart;
1871da177e4SLinus Torvalds }
1881da177e4SLinus Torvalds
1891da177e4SLinus Torvalds if (!dont_cache)
1901da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, ROUTE4_FAILURE);
1911da177e4SLinus Torvalds failure:
1921da177e4SLinus Torvalds return -1;
1931da177e4SLinus Torvalds }
1941da177e4SLinus Torvalds
to_hash(u32 id)1951da177e4SLinus Torvalds static inline u32 to_hash(u32 id)
1961da177e4SLinus Torvalds {
1971da177e4SLinus Torvalds u32 h = id & 0xFF;
198cc7ec456SEric Dumazet
1991da177e4SLinus Torvalds if (id & 0x8000)
2001da177e4SLinus Torvalds h += 256;
2011da177e4SLinus Torvalds return h;
2021da177e4SLinus Torvalds }
2031da177e4SLinus Torvalds
from_hash(u32 id)2041da177e4SLinus Torvalds static inline u32 from_hash(u32 id)
2051da177e4SLinus Torvalds {
2061da177e4SLinus Torvalds id &= 0xFFFF;
2071da177e4SLinus Torvalds if (id == 0xFFFF)
2081da177e4SLinus Torvalds return 32;
2091da177e4SLinus Torvalds if (!(id & 0x8000)) {
2101da177e4SLinus Torvalds if (id > 255)
2111da177e4SLinus Torvalds return 256;
2121da177e4SLinus Torvalds return id & 0xF;
2131da177e4SLinus Torvalds }
2141da177e4SLinus Torvalds return 16 + (id & 0xF);
2151da177e4SLinus Torvalds }
2161da177e4SLinus Torvalds
route4_get(struct tcf_proto * tp,u32 handle)2178113c095SWANG Cong static void *route4_get(struct tcf_proto *tp, u32 handle)
2181da177e4SLinus Torvalds {
2191109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root);
2201da177e4SLinus Torvalds struct route4_bucket *b;
2211da177e4SLinus Torvalds struct route4_filter *f;
222cc7ec456SEric Dumazet unsigned int h1, h2;
2231da177e4SLinus Torvalds
2241da177e4SLinus Torvalds h1 = to_hash(handle);
2251da177e4SLinus Torvalds if (h1 > 256)
2268113c095SWANG Cong return NULL;
2271da177e4SLinus Torvalds
2281da177e4SLinus Torvalds h2 = from_hash(handle >> 16);
2291da177e4SLinus Torvalds if (h2 > 32)
2308113c095SWANG Cong return NULL;
2311da177e4SLinus Torvalds
2321109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]);
233cc7ec456SEric Dumazet if (b) {
2341109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h2]);
2351109c005SJohn Fastabend f;
2361109c005SJohn Fastabend f = rtnl_dereference(f->next))
2371da177e4SLinus Torvalds if (f->handle == handle)
2388113c095SWANG Cong return f;
2391da177e4SLinus Torvalds }
2408113c095SWANG Cong return NULL;
2411da177e4SLinus Torvalds }
2421da177e4SLinus Torvalds
route4_init(struct tcf_proto * tp)2431da177e4SLinus Torvalds static int route4_init(struct tcf_proto *tp)
2441da177e4SLinus Torvalds {
245a05c2d11SWANG Cong struct route4_head *head;
246a05c2d11SWANG Cong
247a05c2d11SWANG Cong head = kzalloc(sizeof(struct route4_head), GFP_KERNEL);
248a05c2d11SWANG Cong if (head == NULL)
249a05c2d11SWANG Cong return -ENOBUFS;
250a05c2d11SWANG Cong
251a05c2d11SWANG Cong rcu_assign_pointer(tp->root, head);
2521da177e4SLinus Torvalds return 0;
2531da177e4SLinus Torvalds }
2541da177e4SLinus Torvalds
__route4_delete_filter(struct route4_filter * f)2553fd51de5SCong Wang static void __route4_delete_filter(struct route4_filter *f)
2563fd51de5SCong Wang {
2573fd51de5SCong Wang tcf_exts_destroy(&f->exts);
2583fd51de5SCong Wang tcf_exts_put_net(&f->exts);
2593fd51de5SCong Wang kfree(f);
2603fd51de5SCong Wang }
2613fd51de5SCong Wang
route4_delete_filter_work(struct work_struct * work)262c2f3f31dSCong Wang static void route4_delete_filter_work(struct work_struct *work)
263c2f3f31dSCong Wang {
264aaa908ffSCong Wang struct route4_filter *f = container_of(to_rcu_work(work),
265aaa908ffSCong Wang struct route4_filter,
266aaa908ffSCong Wang rwork);
267c2f3f31dSCong Wang rtnl_lock();
2683fd51de5SCong Wang __route4_delete_filter(f);
269c2f3f31dSCong Wang rtnl_unlock();
270c2f3f31dSCong Wang }
271c2f3f31dSCong Wang
route4_queue_work(struct route4_filter * f)272aaa908ffSCong Wang static void route4_queue_work(struct route4_filter *f)
2731da177e4SLinus Torvalds {
274aaa908ffSCong Wang tcf_queue_work(&f->rwork, route4_delete_filter_work);
2751da177e4SLinus Torvalds }
2761da177e4SLinus Torvalds
route4_destroy(struct tcf_proto * tp,bool rtnl_held,struct netlink_ext_ack * extack)27712db03b6SVlad Buslov static void route4_destroy(struct tcf_proto *tp, bool rtnl_held,
27812db03b6SVlad Buslov struct netlink_ext_ack *extack)
2791da177e4SLinus Torvalds {
2801109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root);
2811da177e4SLinus Torvalds int h1, h2;
2821da177e4SLinus Torvalds
2831da177e4SLinus Torvalds if (head == NULL)
284763dbf63SWANG Cong return;
2851da177e4SLinus Torvalds
2861da177e4SLinus Torvalds for (h1 = 0; h1 <= 256; h1++) {
2871da177e4SLinus Torvalds struct route4_bucket *b;
2881da177e4SLinus Torvalds
2891109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]);
290cc7ec456SEric Dumazet if (b) {
2911da177e4SLinus Torvalds for (h2 = 0; h2 <= 32; h2++) {
2921da177e4SLinus Torvalds struct route4_filter *f;
2931da177e4SLinus Torvalds
2941109c005SJohn Fastabend while ((f = rtnl_dereference(b->ht[h2])) != NULL) {
2951109c005SJohn Fastabend struct route4_filter *next;
2961109c005SJohn Fastabend
2971109c005SJohn Fastabend next = rtnl_dereference(f->next);
2981109c005SJohn Fastabend RCU_INIT_POINTER(b->ht[h2], next);
29918cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res);
3003fd51de5SCong Wang if (tcf_exts_get_net(&f->exts))
301aaa908ffSCong Wang route4_queue_work(f);
3023fd51de5SCong Wang else
3033fd51de5SCong Wang __route4_delete_filter(f);
3041da177e4SLinus Torvalds }
3051da177e4SLinus Torvalds }
3061109c005SJohn Fastabend RCU_INIT_POINTER(head->table[h1], NULL);
3071109c005SJohn Fastabend kfree_rcu(b, rcu);
3081da177e4SLinus Torvalds }
3091da177e4SLinus Torvalds }
3101109c005SJohn Fastabend kfree_rcu(head, rcu);
3111da177e4SLinus Torvalds }
3121da177e4SLinus Torvalds
route4_delete(struct tcf_proto * tp,void * arg,bool * last,bool rtnl_held,struct netlink_ext_ack * extack)313571acf21SAlexander Aring static int route4_delete(struct tcf_proto *tp, void *arg, bool *last,
31412db03b6SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack)
3151da177e4SLinus Torvalds {
3161109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root);
3178113c095SWANG Cong struct route4_filter *f = arg;
3181109c005SJohn Fastabend struct route4_filter __rcu **fp;
3191109c005SJohn Fastabend struct route4_filter *nf;
3201da177e4SLinus Torvalds struct route4_bucket *b;
3211109c005SJohn Fastabend unsigned int h = 0;
322763dbf63SWANG Cong int i, h1;
3231da177e4SLinus Torvalds
3241da177e4SLinus Torvalds if (!head || !f)
3251da177e4SLinus Torvalds return -EINVAL;
3261da177e4SLinus Torvalds
3271da177e4SLinus Torvalds h = f->handle;
3281da177e4SLinus Torvalds b = f->bkt;
3291da177e4SLinus Torvalds
3301109c005SJohn Fastabend fp = &b->ht[from_hash(h >> 16)];
3311109c005SJohn Fastabend for (nf = rtnl_dereference(*fp); nf;
3321109c005SJohn Fastabend fp = &nf->next, nf = rtnl_dereference(*fp)) {
3331109c005SJohn Fastabend if (nf == f) {
3341109c005SJohn Fastabend /* unlink it */
3351109c005SJohn Fastabend RCU_INIT_POINTER(*fp, rtnl_dereference(f->next));
3361da177e4SLinus Torvalds
3371109c005SJohn Fastabend /* Remove any fastmap lookups that might ref filter
3381109c005SJohn Fastabend * notice we unlink'd the filter so we can't get it
3391109c005SJohn Fastabend * back in the fastmap.
3401109c005SJohn Fastabend */
3411109c005SJohn Fastabend route4_reset_fastmap(head);
3421da177e4SLinus Torvalds
3431109c005SJohn Fastabend /* Delete it */
34418cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res);
3453fd51de5SCong Wang tcf_exts_get_net(&f->exts);
346aaa908ffSCong Wang tcf_queue_work(&f->rwork, route4_delete_filter_work);
3471da177e4SLinus Torvalds
3481109c005SJohn Fastabend /* Strip RTNL protected tree */
3491109c005SJohn Fastabend for (i = 0; i <= 32; i++) {
3501109c005SJohn Fastabend struct route4_filter *rt;
3511109c005SJohn Fastabend
3521109c005SJohn Fastabend rt = rtnl_dereference(b->ht[i]);
3531109c005SJohn Fastabend if (rt)
354763dbf63SWANG Cong goto out;
3551109c005SJohn Fastabend }
3561da177e4SLinus Torvalds
3571da177e4SLinus Torvalds /* OK, session has no flows */
3581109c005SJohn Fastabend RCU_INIT_POINTER(head->table[to_hash(h)], NULL);
3591109c005SJohn Fastabend kfree_rcu(b, rcu);
360763dbf63SWANG Cong break;
361763dbf63SWANG Cong }
362763dbf63SWANG Cong }
3631da177e4SLinus Torvalds
364763dbf63SWANG Cong out:
365763dbf63SWANG Cong *last = true;
366763dbf63SWANG Cong for (h1 = 0; h1 <= 256; h1++) {
367763dbf63SWANG Cong if (rcu_access_pointer(head->table[h1])) {
368763dbf63SWANG Cong *last = false;
369763dbf63SWANG Cong break;
3701da177e4SLinus Torvalds }
3711da177e4SLinus Torvalds }
372763dbf63SWANG Cong
3731da177e4SLinus Torvalds return 0;
3741da177e4SLinus Torvalds }
3751da177e4SLinus Torvalds
3766fa8c014SPatrick McHardy static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = {
3776fa8c014SPatrick McHardy [TCA_ROUTE4_CLASSID] = { .type = NLA_U32 },
3786fa8c014SPatrick McHardy [TCA_ROUTE4_TO] = { .type = NLA_U32 },
3796fa8c014SPatrick McHardy [TCA_ROUTE4_FROM] = { .type = NLA_U32 },
3806fa8c014SPatrick McHardy [TCA_ROUTE4_IIF] = { .type = NLA_U32 },
3816fa8c014SPatrick McHardy };
3826fa8c014SPatrick McHardy
route4_set_parms(struct net * net,struct tcf_proto * tp,unsigned long base,struct route4_filter * f,u32 handle,struct route4_head * head,struct nlattr ** tb,struct nlattr * est,int new,u32 flags,struct netlink_ext_ack * extack)383c1b52739SBenjamin LaHaise static int route4_set_parms(struct net *net, struct tcf_proto *tp,
384c1b52739SBenjamin LaHaise unsigned long base, struct route4_filter *f,
385c1b52739SBenjamin LaHaise u32 handle, struct route4_head *head,
3862f7ef2f8SCong Wang struct nlattr **tb, struct nlattr *est, int new,
387695176bfSCong Wang u32 flags, struct netlink_ext_ack *extack)
3881da177e4SLinus Torvalds {
3891da177e4SLinus Torvalds u32 id = 0, to = 0, nhandle = 0x8000;
3901da177e4SLinus Torvalds struct route4_filter *fp;
3911da177e4SLinus Torvalds unsigned int h1;
3921da177e4SLinus Torvalds struct route4_bucket *b;
393b9a24bb7SWANG Cong int err;
3941da177e4SLinus Torvalds
395695176bfSCong Wang err = tcf_exts_validate(net, tp, tb, est, &f->exts, flags, extack);
3961da177e4SLinus Torvalds if (err < 0)
3971da177e4SLinus Torvalds return err;
3981da177e4SLinus Torvalds
399add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO]) {
4001da177e4SLinus Torvalds if (new && handle & 0x8000)
4018c98d571SJiri Pirko return -EINVAL;
4021587bac4SPatrick McHardy to = nla_get_u32(tb[TCA_ROUTE4_TO]);
4031da177e4SLinus Torvalds if (to > 0xFF)
4048c98d571SJiri Pirko return -EINVAL;
4051da177e4SLinus Torvalds nhandle = to;
4061da177e4SLinus Torvalds }
4071da177e4SLinus Torvalds
408add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM]) {
409add93b61SPatrick McHardy if (tb[TCA_ROUTE4_IIF])
4108c98d571SJiri Pirko return -EINVAL;
4111587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_FROM]);
4121da177e4SLinus Torvalds if (id > 0xFF)
4138c98d571SJiri Pirko return -EINVAL;
4141da177e4SLinus Torvalds nhandle |= id << 16;
415add93b61SPatrick McHardy } else if (tb[TCA_ROUTE4_IIF]) {
4161587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_IIF]);
4171da177e4SLinus Torvalds if (id > 0x7FFF)
4188c98d571SJiri Pirko return -EINVAL;
4191da177e4SLinus Torvalds nhandle |= (id | 0x8000) << 16;
4201da177e4SLinus Torvalds } else
4211da177e4SLinus Torvalds nhandle |= 0xFFFF << 16;
4221da177e4SLinus Torvalds
4231da177e4SLinus Torvalds if (handle && new) {
4241da177e4SLinus Torvalds nhandle |= handle & 0x7F00;
4251da177e4SLinus Torvalds if (nhandle != handle)
4268c98d571SJiri Pirko return -EINVAL;
4271da177e4SLinus Torvalds }
4281da177e4SLinus Torvalds
42902799571SJamal Hadi Salim if (!nhandle) {
43002799571SJamal Hadi Salim NL_SET_ERR_MSG(extack, "Replacing with handle of 0 is invalid");
43102799571SJamal Hadi Salim return -EINVAL;
43202799571SJamal Hadi Salim }
43302799571SJamal Hadi Salim
4341da177e4SLinus Torvalds h1 = to_hash(nhandle);
4351109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]);
436cc7ec456SEric Dumazet if (!b) {
4370da974f4SPanagiotis Issaris b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL);
4381da177e4SLinus Torvalds if (b == NULL)
4398c98d571SJiri Pirko return -ENOBUFS;
4401da177e4SLinus Torvalds
4411109c005SJohn Fastabend rcu_assign_pointer(head->table[h1], b);
4421da177e4SLinus Torvalds } else {
4431da177e4SLinus Torvalds unsigned int h2 = from_hash(nhandle >> 16);
444cc7ec456SEric Dumazet
4451109c005SJohn Fastabend for (fp = rtnl_dereference(b->ht[h2]);
4461109c005SJohn Fastabend fp;
4471109c005SJohn Fastabend fp = rtnl_dereference(fp->next))
4481da177e4SLinus Torvalds if (fp->handle == f->handle)
4498c98d571SJiri Pirko return -EEXIST;
4501da177e4SLinus Torvalds }
4511da177e4SLinus Torvalds
452add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO])
4531da177e4SLinus Torvalds f->id = to;
4541da177e4SLinus Torvalds
455add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM])
4561da177e4SLinus Torvalds f->id = to | id<<16;
457add93b61SPatrick McHardy else if (tb[TCA_ROUTE4_IIF])
4581da177e4SLinus Torvalds f->iif = id;
4591da177e4SLinus Torvalds
4601da177e4SLinus Torvalds f->handle = nhandle;
4611da177e4SLinus Torvalds f->bkt = b;
4621109c005SJohn Fastabend f->tp = tp;
4631da177e4SLinus Torvalds
464add93b61SPatrick McHardy if (tb[TCA_ROUTE4_CLASSID]) {
4651587bac4SPatrick McHardy f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]);
4661da177e4SLinus Torvalds tcf_bind_filter(tp, &f->res, base);
4671da177e4SLinus Torvalds }
4681da177e4SLinus Torvalds
4691da177e4SLinus Torvalds return 0;
4701da177e4SLinus Torvalds }
4711da177e4SLinus Torvalds
route4_change(struct net * net,struct sk_buff * in_skb,struct tcf_proto * tp,unsigned long base,u32 handle,struct nlattr ** tca,void ** arg,u32 flags,struct netlink_ext_ack * extack)472c1b52739SBenjamin LaHaise static int route4_change(struct net *net, struct sk_buff *in_skb,
4735a7a5555SJamal Hadi Salim struct tcf_proto *tp, unsigned long base, u32 handle,
474695176bfSCong Wang struct nlattr **tca, void **arg, u32 flags,
475695176bfSCong Wang struct netlink_ext_ack *extack)
4761da177e4SLinus Torvalds {
4771109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root);
4781109c005SJohn Fastabend struct route4_filter __rcu **fp;
4791109c005SJohn Fastabend struct route4_filter *fold, *f1, *pfp, *f = NULL;
4801da177e4SLinus Torvalds struct route4_bucket *b;
481add93b61SPatrick McHardy struct nlattr *opt = tca[TCA_OPTIONS];
482add93b61SPatrick McHardy struct nlattr *tb[TCA_ROUTE4_MAX + 1];
4831da177e4SLinus Torvalds unsigned int h, th;
4841da177e4SLinus Torvalds int err;
4851109c005SJohn Fastabend bool new = true;
4861da177e4SLinus Torvalds
48702799571SJamal Hadi Salim if (!handle) {
48802799571SJamal Hadi Salim NL_SET_ERR_MSG(extack, "Creating with handle of 0 is invalid");
48902799571SJamal Hadi Salim return -EINVAL;
49002799571SJamal Hadi Salim }
49102799571SJamal Hadi Salim
4921da177e4SLinus Torvalds if (opt == NULL)
49353a40680SDan Carpenter return -EINVAL;
4941da177e4SLinus Torvalds
4958cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_ROUTE4_MAX, opt,
4968cb08174SJohannes Berg route4_policy, NULL);
497cee63723SPatrick McHardy if (err < 0)
498cee63723SPatrick McHardy return err;
4991da177e4SLinus Torvalds
5008113c095SWANG Cong fold = *arg;
50153a40680SDan Carpenter if (fold && fold->handle != handle)
5021da177e4SLinus Torvalds return -EINVAL;
5031da177e4SLinus Torvalds
5041da177e4SLinus Torvalds err = -ENOBUFS;
5050da974f4SPanagiotis Issaris f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL);
5061109c005SJohn Fastabend if (!f)
5071da177e4SLinus Torvalds goto errout;
5081da177e4SLinus Torvalds
50914215108SCong Wang err = tcf_exts_init(&f->exts, net, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE);
510b9a24bb7SWANG Cong if (err < 0)
511b9a24bb7SWANG Cong goto errout;
512b9a24bb7SWANG Cong
5131109c005SJohn Fastabend if (fold) {
5141109c005SJohn Fastabend f->id = fold->id;
5151109c005SJohn Fastabend f->iif = fold->iif;
5161109c005SJohn Fastabend f->handle = fold->handle;
5171109c005SJohn Fastabend
5181109c005SJohn Fastabend f->tp = fold->tp;
5191109c005SJohn Fastabend f->bkt = fold->bkt;
5201109c005SJohn Fastabend new = false;
5211109c005SJohn Fastabend }
5221109c005SJohn Fastabend
523c1b52739SBenjamin LaHaise err = route4_set_parms(net, tp, base, f, handle, head, tb,
524695176bfSCong Wang tca[TCA_RATE], new, flags, extack);
5251da177e4SLinus Torvalds if (err < 0)
5261da177e4SLinus Torvalds goto errout;
5271da177e4SLinus Torvalds
5281da177e4SLinus Torvalds h = from_hash(f->handle >> 16);
5291109c005SJohn Fastabend fp = &f->bkt->ht[h];
5301109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp);
5311109c005SJohn Fastabend (f1 = rtnl_dereference(*fp)) != NULL;
5321109c005SJohn Fastabend fp = &f1->next)
5331da177e4SLinus Torvalds if (f->handle < f1->handle)
5341da177e4SLinus Torvalds break;
5351da177e4SLinus Torvalds
536f36fe1c4SJiri Pirko tcf_block_netif_keep_dst(tp->chain->block);
5371109c005SJohn Fastabend rcu_assign_pointer(f->next, f1);
5381109c005SJohn Fastabend rcu_assign_pointer(*fp, f);
5391da177e4SLinus Torvalds
5409ad36309SThadeu Lima de Souza Cascardo if (fold) {
5411109c005SJohn Fastabend th = to_hash(fold->handle);
5421109c005SJohn Fastabend h = from_hash(fold->handle >> 16);
5431109c005SJohn Fastabend b = rtnl_dereference(head->table[th]);
544cc7ec456SEric Dumazet if (b) {
5451109c005SJohn Fastabend fp = &b->ht[h];
5461109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp); pfp;
5471109c005SJohn Fastabend fp = &pfp->next, pfp = rtnl_dereference(*fp)) {
548ef299cc3SCong Wang if (pfp == fold) {
549ef299cc3SCong Wang rcu_assign_pointer(*fp, fold->next);
5501da177e4SLinus Torvalds break;
5511da177e4SLinus Torvalds }
5521da177e4SLinus Torvalds }
5531da177e4SLinus Torvalds }
5541da177e4SLinus Torvalds }
5551da177e4SLinus Torvalds
5561109c005SJohn Fastabend route4_reset_fastmap(head);
5578113c095SWANG Cong *arg = f;
55818cdb37eSJohn Fastabend if (fold) {
55918cdb37eSJohn Fastabend tcf_unbind_filter(tp, &fold->res);
5603fd51de5SCong Wang tcf_exts_get_net(&fold->exts);
561aaa908ffSCong Wang tcf_queue_work(&fold->rwork, route4_delete_filter_work);
56218cdb37eSJohn Fastabend }
5631da177e4SLinus Torvalds return 0;
5641da177e4SLinus Torvalds
5651da177e4SLinus Torvalds errout:
56621641c2eSWANG Cong if (f)
567b9a24bb7SWANG Cong tcf_exts_destroy(&f->exts);
5681da177e4SLinus Torvalds kfree(f);
5691da177e4SLinus Torvalds return err;
5701da177e4SLinus Torvalds }
5711da177e4SLinus Torvalds
route4_walk(struct tcf_proto * tp,struct tcf_walker * arg,bool rtnl_held)57212db03b6SVlad Buslov static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg,
57312db03b6SVlad Buslov bool rtnl_held)
5741da177e4SLinus Torvalds {
5751109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root);
576cc7ec456SEric Dumazet unsigned int h, h1;
5771da177e4SLinus Torvalds
5783027ff41SVlad Buslov if (head == NULL || arg->stop)
5791da177e4SLinus Torvalds return;
5801da177e4SLinus Torvalds
5811da177e4SLinus Torvalds for (h = 0; h <= 256; h++) {
5821109c005SJohn Fastabend struct route4_bucket *b = rtnl_dereference(head->table[h]);
5831da177e4SLinus Torvalds
5841da177e4SLinus Torvalds if (b) {
5851da177e4SLinus Torvalds for (h1 = 0; h1 <= 32; h1++) {
5861da177e4SLinus Torvalds struct route4_filter *f;
5871da177e4SLinus Torvalds
5881109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h1]);
5891109c005SJohn Fastabend f;
5901109c005SJohn Fastabend f = rtnl_dereference(f->next)) {
5915508ff7cSZhengchao Shao if (!tc_cls_stats_dump(tp, arg, f))
5921da177e4SLinus Torvalds return;
5931da177e4SLinus Torvalds }
5941da177e4SLinus Torvalds }
5951da177e4SLinus Torvalds }
5961da177e4SLinus Torvalds }
5971da177e4SLinus Torvalds }
5981da177e4SLinus Torvalds
route4_dump(struct net * net,struct tcf_proto * tp,void * fh,struct sk_buff * skb,struct tcmsg * t,bool rtnl_held)5998113c095SWANG Cong static int route4_dump(struct net *net, struct tcf_proto *tp, void *fh,
60012db03b6SVlad Buslov struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
6011da177e4SLinus Torvalds {
6028113c095SWANG Cong struct route4_filter *f = fh;
6034b3550efSPatrick McHardy struct nlattr *nest;
6041da177e4SLinus Torvalds u32 id;
6051da177e4SLinus Torvalds
6061da177e4SLinus Torvalds if (f == NULL)
6071da177e4SLinus Torvalds return skb->len;
6081da177e4SLinus Torvalds
6091da177e4SLinus Torvalds t->tcm_handle = f->handle;
6101da177e4SLinus Torvalds
611ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
6124b3550efSPatrick McHardy if (nest == NULL)
6134b3550efSPatrick McHardy goto nla_put_failure;
6141da177e4SLinus Torvalds
6151da177e4SLinus Torvalds if (!(f->handle & 0x8000)) {
6161da177e4SLinus Torvalds id = f->id & 0xFF;
6171b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_TO, id))
6181b34ec43SDavid S. Miller goto nla_put_failure;
6191da177e4SLinus Torvalds }
6201da177e4SLinus Torvalds if (f->handle & 0x80000000) {
6211b34ec43SDavid S. Miller if ((f->handle >> 16) != 0xFFFF &&
6221b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_IIF, f->iif))
6231b34ec43SDavid S. Miller goto nla_put_failure;
6241da177e4SLinus Torvalds } else {
6251da177e4SLinus Torvalds id = f->id >> 16;
6261b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_FROM, id))
6271b34ec43SDavid S. Miller goto nla_put_failure;
6281da177e4SLinus Torvalds }
6291b34ec43SDavid S. Miller if (f->res.classid &&
6301b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_CLASSID, f->res.classid))
6311b34ec43SDavid S. Miller goto nla_put_failure;
6321da177e4SLinus Torvalds
6335da57f42SWANG Cong if (tcf_exts_dump(skb, &f->exts) < 0)
634add93b61SPatrick McHardy goto nla_put_failure;
6351da177e4SLinus Torvalds
6364b3550efSPatrick McHardy nla_nest_end(skb, nest);
6371da177e4SLinus Torvalds
6385da57f42SWANG Cong if (tcf_exts_dump_stats(skb, &f->exts) < 0)
639add93b61SPatrick McHardy goto nla_put_failure;
6401da177e4SLinus Torvalds
6411da177e4SLinus Torvalds return skb->len;
6421da177e4SLinus Torvalds
643add93b61SPatrick McHardy nla_put_failure:
6446ea3b446SJiri Pirko nla_nest_cancel(skb, nest);
6451da177e4SLinus Torvalds return -1;
6461da177e4SLinus Torvalds }
6471da177e4SLinus Torvalds
route4_bind_class(void * fh,u32 classid,unsigned long cl,void * q,unsigned long base)6482e24cd75SCong Wang static void route4_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
6492e24cd75SCong Wang unsigned long base)
65007d79fc7SCong Wang {
65107d79fc7SCong Wang struct route4_filter *f = fh;
65207d79fc7SCong Wang
653cc9039a1SZhengchao Shao tc_cls_bind_class(classid, cl, q, &f->res, base);
65407d79fc7SCong Wang }
65507d79fc7SCong Wang
6562eb9d75cSPatrick McHardy static struct tcf_proto_ops cls_route4_ops __read_mostly = {
6571da177e4SLinus Torvalds .kind = "route",
6581da177e4SLinus Torvalds .classify = route4_classify,
6591da177e4SLinus Torvalds .init = route4_init,
6601da177e4SLinus Torvalds .destroy = route4_destroy,
6611da177e4SLinus Torvalds .get = route4_get,
6621da177e4SLinus Torvalds .change = route4_change,
6631da177e4SLinus Torvalds .delete = route4_delete,
6641da177e4SLinus Torvalds .walk = route4_walk,
6651da177e4SLinus Torvalds .dump = route4_dump,
66607d79fc7SCong Wang .bind_class = route4_bind_class,
6671da177e4SLinus Torvalds .owner = THIS_MODULE,
6681da177e4SLinus Torvalds };
6691da177e4SLinus Torvalds
init_route4(void)6701da177e4SLinus Torvalds static int __init init_route4(void)
6711da177e4SLinus Torvalds {
6721da177e4SLinus Torvalds return register_tcf_proto_ops(&cls_route4_ops);
6731da177e4SLinus Torvalds }
6741da177e4SLinus Torvalds
exit_route4(void)6751da177e4SLinus Torvalds static void __exit exit_route4(void)
6761da177e4SLinus Torvalds {
6771da177e4SLinus Torvalds unregister_tcf_proto_ops(&cls_route4_ops);
6781da177e4SLinus Torvalds }
6791da177e4SLinus Torvalds
6801da177e4SLinus Torvalds module_init(init_route4)
6811da177e4SLinus Torvalds module_exit(exit_route4)
6821da177e4SLinus Torvalds MODULE_LICENSE("GPL");
683