12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * net/sched/cls_route.c ROUTE4 classifier. 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds 81da177e4SLinus Torvalds #include <linux/module.h> 95a0e3ad6STejun Heo #include <linux/slab.h> 101da177e4SLinus Torvalds #include <linux/types.h> 111da177e4SLinus Torvalds #include <linux/kernel.h> 121da177e4SLinus Torvalds #include <linux/string.h> 131da177e4SLinus Torvalds #include <linux/errno.h> 141da177e4SLinus Torvalds #include <linux/skbuff.h> 150ba48053SPatrick McHardy #include <net/dst.h> 160ba48053SPatrick McHardy #include <net/route.h> 170ba48053SPatrick McHardy #include <net/netlink.h> 181da177e4SLinus Torvalds #include <net/act_api.h> 191da177e4SLinus Torvalds #include <net/pkt_cls.h> 201da177e4SLinus Torvalds 211da177e4SLinus Torvalds /* 22cc7ec456SEric Dumazet * 1. For now we assume that route tags < 256. 23cc7ec456SEric Dumazet * It allows to use direct table lookups, instead of hash tables. 24cc7ec456SEric Dumazet * 2. For now we assume that "from TAG" and "fromdev DEV" statements 25cc7ec456SEric Dumazet * are mutually exclusive. 26cc7ec456SEric Dumazet * 3. "to TAG from ANY" has higher priority, than "to ANY from XXX" 271da177e4SLinus Torvalds */ 28cc7ec456SEric Dumazet struct route4_fastmap { 291da177e4SLinus Torvalds struct route4_filter *filter; 301da177e4SLinus Torvalds u32 id; 311da177e4SLinus Torvalds int iif; 321da177e4SLinus Torvalds }; 331da177e4SLinus Torvalds 34cc7ec456SEric Dumazet struct route4_head { 351da177e4SLinus Torvalds struct route4_fastmap fastmap[16]; 361109c005SJohn Fastabend struct route4_bucket __rcu *table[256 + 1]; 371109c005SJohn Fastabend struct rcu_head rcu; 381da177e4SLinus Torvalds }; 391da177e4SLinus Torvalds 40cc7ec456SEric Dumazet struct route4_bucket { 411da177e4SLinus Torvalds /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */ 421109c005SJohn Fastabend struct route4_filter __rcu *ht[16 + 16 + 1]; 431109c005SJohn Fastabend struct rcu_head rcu; 441da177e4SLinus Torvalds }; 451da177e4SLinus Torvalds 46cc7ec456SEric Dumazet struct route4_filter { 471109c005SJohn Fastabend struct route4_filter __rcu *next; 481da177e4SLinus Torvalds u32 id; 491da177e4SLinus Torvalds int iif; 501da177e4SLinus Torvalds 511da177e4SLinus Torvalds struct tcf_result res; 521da177e4SLinus Torvalds struct tcf_exts exts; 531da177e4SLinus Torvalds u32 handle; 541da177e4SLinus Torvalds struct route4_bucket *bkt; 551109c005SJohn Fastabend struct tcf_proto *tp; 56aaa908ffSCong Wang struct rcu_work rwork; 57c2f3f31dSCong Wang }; 581da177e4SLinus Torvalds 591da177e4SLinus Torvalds #define ROUTE4_FAILURE ((struct route4_filter *)(-1L)) 601da177e4SLinus Torvalds 61cc7ec456SEric Dumazet static inline int route4_fastmap_hash(u32 id, int iif) 621da177e4SLinus Torvalds { 631da177e4SLinus Torvalds return id & 0xF; 641da177e4SLinus Torvalds } 651da177e4SLinus Torvalds 661109c005SJohn Fastabend static DEFINE_SPINLOCK(fastmap_lock); 67cc7ec456SEric Dumazet static void 681109c005SJohn Fastabend route4_reset_fastmap(struct route4_head *head) 691da177e4SLinus Torvalds { 701109c005SJohn Fastabend spin_lock_bh(&fastmap_lock); 711da177e4SLinus Torvalds memset(head->fastmap, 0, sizeof(head->fastmap)); 721109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock); 731da177e4SLinus Torvalds } 741da177e4SLinus Torvalds 75cc7ec456SEric Dumazet static void 761da177e4SLinus Torvalds route4_set_fastmap(struct route4_head *head, u32 id, int iif, 771da177e4SLinus Torvalds struct route4_filter *f) 781da177e4SLinus Torvalds { 791da177e4SLinus Torvalds int h = route4_fastmap_hash(id, iif); 80cc7ec456SEric Dumazet 811109c005SJohn Fastabend /* fastmap updates must look atomic to aling id, iff, filter */ 821109c005SJohn Fastabend spin_lock_bh(&fastmap_lock); 831da177e4SLinus Torvalds head->fastmap[h].id = id; 841da177e4SLinus Torvalds head->fastmap[h].iif = iif; 851da177e4SLinus Torvalds head->fastmap[h].filter = f; 861109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock); 871da177e4SLinus Torvalds } 881da177e4SLinus Torvalds 89cc7ec456SEric Dumazet static inline int route4_hash_to(u32 id) 901da177e4SLinus Torvalds { 911da177e4SLinus Torvalds return id & 0xFF; 921da177e4SLinus Torvalds } 931da177e4SLinus Torvalds 94cc7ec456SEric Dumazet static inline int route4_hash_from(u32 id) 951da177e4SLinus Torvalds { 961da177e4SLinus Torvalds return (id >> 16) & 0xF; 971da177e4SLinus Torvalds } 981da177e4SLinus Torvalds 99cc7ec456SEric Dumazet static inline int route4_hash_iif(int iif) 1001da177e4SLinus Torvalds { 1011da177e4SLinus Torvalds return 16 + ((iif >> 16) & 0xF); 1021da177e4SLinus Torvalds } 1031da177e4SLinus Torvalds 104cc7ec456SEric Dumazet static inline int route4_hash_wild(void) 1051da177e4SLinus Torvalds { 1061da177e4SLinus Torvalds return 32; 1071da177e4SLinus Torvalds } 1081da177e4SLinus Torvalds 1091da177e4SLinus Torvalds #define ROUTE4_APPLY_RESULT() \ 1101da177e4SLinus Torvalds { \ 1111da177e4SLinus Torvalds *res = f->res; \ 1126fc6d06eSJiri Pirko if (tcf_exts_has_actions(&f->exts)) { \ 1131da177e4SLinus Torvalds int r = tcf_exts_exec(skb, &f->exts, res); \ 1141da177e4SLinus Torvalds if (r < 0) { \ 1151da177e4SLinus Torvalds dont_cache = 1; \ 1161da177e4SLinus Torvalds continue; \ 1171da177e4SLinus Torvalds } \ 1181da177e4SLinus Torvalds return r; \ 1191da177e4SLinus Torvalds } else if (!dont_cache) \ 1201da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, f); \ 1211da177e4SLinus Torvalds return 0; \ 1221da177e4SLinus Torvalds } 1231da177e4SLinus Torvalds 124dc7f9f6eSEric Dumazet static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp, 1251da177e4SLinus Torvalds struct tcf_result *res) 1261da177e4SLinus Torvalds { 1271109c005SJohn Fastabend struct route4_head *head = rcu_dereference_bh(tp->root); 1281da177e4SLinus Torvalds struct dst_entry *dst; 1291da177e4SLinus Torvalds struct route4_bucket *b; 1301da177e4SLinus Torvalds struct route4_filter *f; 1311da177e4SLinus Torvalds u32 id, h; 1321da177e4SLinus Torvalds int iif, dont_cache = 0; 1331da177e4SLinus Torvalds 134cc7ec456SEric Dumazet dst = skb_dst(skb); 135cc7ec456SEric Dumazet if (!dst) 1361da177e4SLinus Torvalds goto failure; 1371da177e4SLinus Torvalds 1381da177e4SLinus Torvalds id = dst->tclassid; 1391da177e4SLinus Torvalds 14092101b3bSDavid S. Miller iif = inet_iif(skb); 1411da177e4SLinus Torvalds 1421da177e4SLinus Torvalds h = route4_fastmap_hash(id, iif); 1431109c005SJohn Fastabend 1441109c005SJohn Fastabend spin_lock(&fastmap_lock); 1451da177e4SLinus Torvalds if (id == head->fastmap[h].id && 1461da177e4SLinus Torvalds iif == head->fastmap[h].iif && 1471da177e4SLinus Torvalds (f = head->fastmap[h].filter) != NULL) { 1481109c005SJohn Fastabend if (f == ROUTE4_FAILURE) { 1491109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1501da177e4SLinus Torvalds goto failure; 1511109c005SJohn Fastabend } 1521da177e4SLinus Torvalds 1531da177e4SLinus Torvalds *res = f->res; 1541109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1551da177e4SLinus Torvalds return 0; 1561da177e4SLinus Torvalds } 1571109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1581da177e4SLinus Torvalds 1591da177e4SLinus Torvalds h = route4_hash_to(id); 1601da177e4SLinus Torvalds 1611da177e4SLinus Torvalds restart: 1621109c005SJohn Fastabend b = rcu_dereference_bh(head->table[h]); 163cc7ec456SEric Dumazet if (b) { 1641109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]); 1651109c005SJohn Fastabend f; 1661109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1671da177e4SLinus Torvalds if (f->id == id) 1681da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1691da177e4SLinus Torvalds 1701109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]); 1711109c005SJohn Fastabend f; 1721109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1731da177e4SLinus Torvalds if (f->iif == iif) 1741da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1751da177e4SLinus Torvalds 1761109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]); 1771109c005SJohn Fastabend f; 1781109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1791da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1801da177e4SLinus Torvalds } 1811da177e4SLinus Torvalds if (h < 256) { 1821da177e4SLinus Torvalds h = 256; 1831da177e4SLinus Torvalds id &= ~0xFFFF; 1841da177e4SLinus Torvalds goto restart; 1851da177e4SLinus Torvalds } 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds if (!dont_cache) 1881da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, ROUTE4_FAILURE); 1891da177e4SLinus Torvalds failure: 1901da177e4SLinus Torvalds return -1; 1911da177e4SLinus Torvalds } 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds static inline u32 to_hash(u32 id) 1941da177e4SLinus Torvalds { 1951da177e4SLinus Torvalds u32 h = id & 0xFF; 196cc7ec456SEric Dumazet 1971da177e4SLinus Torvalds if (id & 0x8000) 1981da177e4SLinus Torvalds h += 256; 1991da177e4SLinus Torvalds return h; 2001da177e4SLinus Torvalds } 2011da177e4SLinus Torvalds 2021da177e4SLinus Torvalds static inline u32 from_hash(u32 id) 2031da177e4SLinus Torvalds { 2041da177e4SLinus Torvalds id &= 0xFFFF; 2051da177e4SLinus Torvalds if (id == 0xFFFF) 2061da177e4SLinus Torvalds return 32; 2071da177e4SLinus Torvalds if (!(id & 0x8000)) { 2081da177e4SLinus Torvalds if (id > 255) 2091da177e4SLinus Torvalds return 256; 2101da177e4SLinus Torvalds return id & 0xF; 2111da177e4SLinus Torvalds } 2121da177e4SLinus Torvalds return 16 + (id & 0xF); 2131da177e4SLinus Torvalds } 2141da177e4SLinus Torvalds 2158113c095SWANG Cong static void *route4_get(struct tcf_proto *tp, u32 handle) 2161da177e4SLinus Torvalds { 2171109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 2181da177e4SLinus Torvalds struct route4_bucket *b; 2191da177e4SLinus Torvalds struct route4_filter *f; 220cc7ec456SEric Dumazet unsigned int h1, h2; 2211da177e4SLinus Torvalds 2221da177e4SLinus Torvalds h1 = to_hash(handle); 2231da177e4SLinus Torvalds if (h1 > 256) 2248113c095SWANG Cong return NULL; 2251da177e4SLinus Torvalds 2261da177e4SLinus Torvalds h2 = from_hash(handle >> 16); 2271da177e4SLinus Torvalds if (h2 > 32) 2288113c095SWANG Cong return NULL; 2291da177e4SLinus Torvalds 2301109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 231cc7ec456SEric Dumazet if (b) { 2321109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h2]); 2331109c005SJohn Fastabend f; 2341109c005SJohn Fastabend f = rtnl_dereference(f->next)) 2351da177e4SLinus Torvalds if (f->handle == handle) 2368113c095SWANG Cong return f; 2371da177e4SLinus Torvalds } 2388113c095SWANG Cong return NULL; 2391da177e4SLinus Torvalds } 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds static int route4_init(struct tcf_proto *tp) 2421da177e4SLinus Torvalds { 243a05c2d11SWANG Cong struct route4_head *head; 244a05c2d11SWANG Cong 245a05c2d11SWANG Cong head = kzalloc(sizeof(struct route4_head), GFP_KERNEL); 246a05c2d11SWANG Cong if (head == NULL) 247a05c2d11SWANG Cong return -ENOBUFS; 248a05c2d11SWANG Cong 249a05c2d11SWANG Cong rcu_assign_pointer(tp->root, head); 2501da177e4SLinus Torvalds return 0; 2511da177e4SLinus Torvalds } 2521da177e4SLinus Torvalds 2533fd51de5SCong Wang static void __route4_delete_filter(struct route4_filter *f) 2543fd51de5SCong Wang { 2553fd51de5SCong Wang tcf_exts_destroy(&f->exts); 2563fd51de5SCong Wang tcf_exts_put_net(&f->exts); 2573fd51de5SCong Wang kfree(f); 2583fd51de5SCong Wang } 2593fd51de5SCong Wang 260c2f3f31dSCong Wang static void route4_delete_filter_work(struct work_struct *work) 261c2f3f31dSCong Wang { 262aaa908ffSCong Wang struct route4_filter *f = container_of(to_rcu_work(work), 263aaa908ffSCong Wang struct route4_filter, 264aaa908ffSCong Wang rwork); 265c2f3f31dSCong Wang rtnl_lock(); 2663fd51de5SCong Wang __route4_delete_filter(f); 267c2f3f31dSCong Wang rtnl_unlock(); 268c2f3f31dSCong Wang } 269c2f3f31dSCong Wang 270aaa908ffSCong Wang static void route4_queue_work(struct route4_filter *f) 2711da177e4SLinus Torvalds { 272aaa908ffSCong Wang tcf_queue_work(&f->rwork, route4_delete_filter_work); 2731da177e4SLinus Torvalds } 2741da177e4SLinus Torvalds 27512db03b6SVlad Buslov static void route4_destroy(struct tcf_proto *tp, bool rtnl_held, 27612db03b6SVlad Buslov struct netlink_ext_ack *extack) 2771da177e4SLinus Torvalds { 2781109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 2791da177e4SLinus Torvalds int h1, h2; 2801da177e4SLinus Torvalds 2811da177e4SLinus Torvalds if (head == NULL) 282763dbf63SWANG Cong return; 2831da177e4SLinus Torvalds 2841da177e4SLinus Torvalds for (h1 = 0; h1 <= 256; h1++) { 2851da177e4SLinus Torvalds struct route4_bucket *b; 2861da177e4SLinus Torvalds 2871109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 288cc7ec456SEric Dumazet if (b) { 2891da177e4SLinus Torvalds for (h2 = 0; h2 <= 32; h2++) { 2901da177e4SLinus Torvalds struct route4_filter *f; 2911da177e4SLinus Torvalds 2921109c005SJohn Fastabend while ((f = rtnl_dereference(b->ht[h2])) != NULL) { 2931109c005SJohn Fastabend struct route4_filter *next; 2941109c005SJohn Fastabend 2951109c005SJohn Fastabend next = rtnl_dereference(f->next); 2961109c005SJohn Fastabend RCU_INIT_POINTER(b->ht[h2], next); 29718cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res); 2983fd51de5SCong Wang if (tcf_exts_get_net(&f->exts)) 299aaa908ffSCong Wang route4_queue_work(f); 3003fd51de5SCong Wang else 3013fd51de5SCong Wang __route4_delete_filter(f); 3021da177e4SLinus Torvalds } 3031da177e4SLinus Torvalds } 3041109c005SJohn Fastabend RCU_INIT_POINTER(head->table[h1], NULL); 3051109c005SJohn Fastabend kfree_rcu(b, rcu); 3061da177e4SLinus Torvalds } 3071da177e4SLinus Torvalds } 3081109c005SJohn Fastabend kfree_rcu(head, rcu); 3091da177e4SLinus Torvalds } 3101da177e4SLinus Torvalds 311571acf21SAlexander Aring static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, 31212db03b6SVlad Buslov bool rtnl_held, struct netlink_ext_ack *extack) 3131da177e4SLinus Torvalds { 3141109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 3158113c095SWANG Cong struct route4_filter *f = arg; 3161109c005SJohn Fastabend struct route4_filter __rcu **fp; 3171109c005SJohn Fastabend struct route4_filter *nf; 3181da177e4SLinus Torvalds struct route4_bucket *b; 3191109c005SJohn Fastabend unsigned int h = 0; 320763dbf63SWANG Cong int i, h1; 3211da177e4SLinus Torvalds 3221da177e4SLinus Torvalds if (!head || !f) 3231da177e4SLinus Torvalds return -EINVAL; 3241da177e4SLinus Torvalds 3251da177e4SLinus Torvalds h = f->handle; 3261da177e4SLinus Torvalds b = f->bkt; 3271da177e4SLinus Torvalds 3281109c005SJohn Fastabend fp = &b->ht[from_hash(h >> 16)]; 3291109c005SJohn Fastabend for (nf = rtnl_dereference(*fp); nf; 3301109c005SJohn Fastabend fp = &nf->next, nf = rtnl_dereference(*fp)) { 3311109c005SJohn Fastabend if (nf == f) { 3321109c005SJohn Fastabend /* unlink it */ 3331109c005SJohn Fastabend RCU_INIT_POINTER(*fp, rtnl_dereference(f->next)); 3341da177e4SLinus Torvalds 3351109c005SJohn Fastabend /* Remove any fastmap lookups that might ref filter 3361109c005SJohn Fastabend * notice we unlink'd the filter so we can't get it 3371109c005SJohn Fastabend * back in the fastmap. 3381109c005SJohn Fastabend */ 3391109c005SJohn Fastabend route4_reset_fastmap(head); 3401da177e4SLinus Torvalds 3411109c005SJohn Fastabend /* Delete it */ 34218cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res); 3433fd51de5SCong Wang tcf_exts_get_net(&f->exts); 344aaa908ffSCong Wang tcf_queue_work(&f->rwork, route4_delete_filter_work); 3451da177e4SLinus Torvalds 3461109c005SJohn Fastabend /* Strip RTNL protected tree */ 3471109c005SJohn Fastabend for (i = 0; i <= 32; i++) { 3481109c005SJohn Fastabend struct route4_filter *rt; 3491109c005SJohn Fastabend 3501109c005SJohn Fastabend rt = rtnl_dereference(b->ht[i]); 3511109c005SJohn Fastabend if (rt) 352763dbf63SWANG Cong goto out; 3531109c005SJohn Fastabend } 3541da177e4SLinus Torvalds 3551da177e4SLinus Torvalds /* OK, session has no flows */ 3561109c005SJohn Fastabend RCU_INIT_POINTER(head->table[to_hash(h)], NULL); 3571109c005SJohn Fastabend kfree_rcu(b, rcu); 358763dbf63SWANG Cong break; 359763dbf63SWANG Cong } 360763dbf63SWANG Cong } 3611da177e4SLinus Torvalds 362763dbf63SWANG Cong out: 363763dbf63SWANG Cong *last = true; 364763dbf63SWANG Cong for (h1 = 0; h1 <= 256; h1++) { 365763dbf63SWANG Cong if (rcu_access_pointer(head->table[h1])) { 366763dbf63SWANG Cong *last = false; 367763dbf63SWANG Cong break; 3681da177e4SLinus Torvalds } 3691da177e4SLinus Torvalds } 370763dbf63SWANG Cong 3711da177e4SLinus Torvalds return 0; 3721da177e4SLinus Torvalds } 3731da177e4SLinus Torvalds 3746fa8c014SPatrick McHardy static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = { 3756fa8c014SPatrick McHardy [TCA_ROUTE4_CLASSID] = { .type = NLA_U32 }, 3766fa8c014SPatrick McHardy [TCA_ROUTE4_TO] = { .type = NLA_U32 }, 3776fa8c014SPatrick McHardy [TCA_ROUTE4_FROM] = { .type = NLA_U32 }, 3786fa8c014SPatrick McHardy [TCA_ROUTE4_IIF] = { .type = NLA_U32 }, 3796fa8c014SPatrick McHardy }; 3806fa8c014SPatrick McHardy 381c1b52739SBenjamin LaHaise static int route4_set_parms(struct net *net, struct tcf_proto *tp, 382c1b52739SBenjamin LaHaise unsigned long base, struct route4_filter *f, 383c1b52739SBenjamin LaHaise u32 handle, struct route4_head *head, 3842f7ef2f8SCong Wang struct nlattr **tb, struct nlattr *est, int new, 385695176bfSCong Wang u32 flags, struct netlink_ext_ack *extack) 3861da177e4SLinus Torvalds { 3871da177e4SLinus Torvalds u32 id = 0, to = 0, nhandle = 0x8000; 3881da177e4SLinus Torvalds struct route4_filter *fp; 3891da177e4SLinus Torvalds unsigned int h1; 3901da177e4SLinus Torvalds struct route4_bucket *b; 391b9a24bb7SWANG Cong int err; 3921da177e4SLinus Torvalds 393695176bfSCong Wang err = tcf_exts_validate(net, tp, tb, est, &f->exts, flags, extack); 3941da177e4SLinus Torvalds if (err < 0) 3951da177e4SLinus Torvalds return err; 3961da177e4SLinus Torvalds 397add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO]) { 3981da177e4SLinus Torvalds if (new && handle & 0x8000) 3998c98d571SJiri Pirko return -EINVAL; 4001587bac4SPatrick McHardy to = nla_get_u32(tb[TCA_ROUTE4_TO]); 4011da177e4SLinus Torvalds if (to > 0xFF) 4028c98d571SJiri Pirko return -EINVAL; 4031da177e4SLinus Torvalds nhandle = to; 4041da177e4SLinus Torvalds } 4051da177e4SLinus Torvalds 406add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM]) { 407add93b61SPatrick McHardy if (tb[TCA_ROUTE4_IIF]) 4088c98d571SJiri Pirko return -EINVAL; 4091587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_FROM]); 4101da177e4SLinus Torvalds if (id > 0xFF) 4118c98d571SJiri Pirko return -EINVAL; 4121da177e4SLinus Torvalds nhandle |= id << 16; 413add93b61SPatrick McHardy } else if (tb[TCA_ROUTE4_IIF]) { 4141587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_IIF]); 4151da177e4SLinus Torvalds if (id > 0x7FFF) 4168c98d571SJiri Pirko return -EINVAL; 4171da177e4SLinus Torvalds nhandle |= (id | 0x8000) << 16; 4181da177e4SLinus Torvalds } else 4191da177e4SLinus Torvalds nhandle |= 0xFFFF << 16; 4201da177e4SLinus Torvalds 4211da177e4SLinus Torvalds if (handle && new) { 4221da177e4SLinus Torvalds nhandle |= handle & 0x7F00; 4231da177e4SLinus Torvalds if (nhandle != handle) 4248c98d571SJiri Pirko return -EINVAL; 4251da177e4SLinus Torvalds } 4261da177e4SLinus Torvalds 4271da177e4SLinus Torvalds h1 = to_hash(nhandle); 4281109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 429cc7ec456SEric Dumazet if (!b) { 4300da974f4SPanagiotis Issaris b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL); 4311da177e4SLinus Torvalds if (b == NULL) 4328c98d571SJiri Pirko return -ENOBUFS; 4331da177e4SLinus Torvalds 4341109c005SJohn Fastabend rcu_assign_pointer(head->table[h1], b); 4351da177e4SLinus Torvalds } else { 4361da177e4SLinus Torvalds unsigned int h2 = from_hash(nhandle >> 16); 437cc7ec456SEric Dumazet 4381109c005SJohn Fastabend for (fp = rtnl_dereference(b->ht[h2]); 4391109c005SJohn Fastabend fp; 4401109c005SJohn Fastabend fp = rtnl_dereference(fp->next)) 4411da177e4SLinus Torvalds if (fp->handle == f->handle) 4428c98d571SJiri Pirko return -EEXIST; 4431da177e4SLinus Torvalds } 4441da177e4SLinus Torvalds 445add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO]) 4461da177e4SLinus Torvalds f->id = to; 4471da177e4SLinus Torvalds 448add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM]) 4491da177e4SLinus Torvalds f->id = to | id<<16; 450add93b61SPatrick McHardy else if (tb[TCA_ROUTE4_IIF]) 4511da177e4SLinus Torvalds f->iif = id; 4521da177e4SLinus Torvalds 4531da177e4SLinus Torvalds f->handle = nhandle; 4541da177e4SLinus Torvalds f->bkt = b; 4551109c005SJohn Fastabend f->tp = tp; 4561da177e4SLinus Torvalds 457add93b61SPatrick McHardy if (tb[TCA_ROUTE4_CLASSID]) { 4581587bac4SPatrick McHardy f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]); 4591da177e4SLinus Torvalds tcf_bind_filter(tp, &f->res, base); 4601da177e4SLinus Torvalds } 4611da177e4SLinus Torvalds 4621da177e4SLinus Torvalds return 0; 4631da177e4SLinus Torvalds } 4641da177e4SLinus Torvalds 465c1b52739SBenjamin LaHaise static int route4_change(struct net *net, struct sk_buff *in_skb, 4665a7a5555SJamal Hadi Salim struct tcf_proto *tp, unsigned long base, u32 handle, 467695176bfSCong Wang struct nlattr **tca, void **arg, u32 flags, 468695176bfSCong Wang struct netlink_ext_ack *extack) 4691da177e4SLinus Torvalds { 4701109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 4711109c005SJohn Fastabend struct route4_filter __rcu **fp; 4721109c005SJohn Fastabend struct route4_filter *fold, *f1, *pfp, *f = NULL; 4731da177e4SLinus Torvalds struct route4_bucket *b; 474add93b61SPatrick McHardy struct nlattr *opt = tca[TCA_OPTIONS]; 475add93b61SPatrick McHardy struct nlattr *tb[TCA_ROUTE4_MAX + 1]; 4761da177e4SLinus Torvalds unsigned int h, th; 4771da177e4SLinus Torvalds int err; 4781109c005SJohn Fastabend bool new = true; 4791da177e4SLinus Torvalds 4801da177e4SLinus Torvalds if (opt == NULL) 4811da177e4SLinus Torvalds return handle ? -EINVAL : 0; 4821da177e4SLinus Torvalds 4838cb08174SJohannes Berg err = nla_parse_nested_deprecated(tb, TCA_ROUTE4_MAX, opt, 4848cb08174SJohannes Berg route4_policy, NULL); 485cee63723SPatrick McHardy if (err < 0) 486cee63723SPatrick McHardy return err; 4871da177e4SLinus Torvalds 4888113c095SWANG Cong fold = *arg; 4891109c005SJohn Fastabend if (fold && handle && fold->handle != handle) 4901da177e4SLinus Torvalds return -EINVAL; 4911da177e4SLinus Torvalds 4921da177e4SLinus Torvalds err = -ENOBUFS; 4930da974f4SPanagiotis Issaris f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL); 4941109c005SJohn Fastabend if (!f) 4951da177e4SLinus Torvalds goto errout; 4961da177e4SLinus Torvalds 49714215108SCong Wang err = tcf_exts_init(&f->exts, net, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE); 498b9a24bb7SWANG Cong if (err < 0) 499b9a24bb7SWANG Cong goto errout; 500b9a24bb7SWANG Cong 5011109c005SJohn Fastabend if (fold) { 5021109c005SJohn Fastabend f->id = fold->id; 5031109c005SJohn Fastabend f->iif = fold->iif; 5041109c005SJohn Fastabend f->res = fold->res; 5051109c005SJohn Fastabend f->handle = fold->handle; 5061109c005SJohn Fastabend 5071109c005SJohn Fastabend f->tp = fold->tp; 5081109c005SJohn Fastabend f->bkt = fold->bkt; 5091109c005SJohn Fastabend new = false; 5101109c005SJohn Fastabend } 5111109c005SJohn Fastabend 512c1b52739SBenjamin LaHaise err = route4_set_parms(net, tp, base, f, handle, head, tb, 513695176bfSCong Wang tca[TCA_RATE], new, flags, extack); 5141da177e4SLinus Torvalds if (err < 0) 5151da177e4SLinus Torvalds goto errout; 5161da177e4SLinus Torvalds 5171da177e4SLinus Torvalds h = from_hash(f->handle >> 16); 5181109c005SJohn Fastabend fp = &f->bkt->ht[h]; 5191109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp); 5201109c005SJohn Fastabend (f1 = rtnl_dereference(*fp)) != NULL; 5211109c005SJohn Fastabend fp = &f1->next) 5221da177e4SLinus Torvalds if (f->handle < f1->handle) 5231da177e4SLinus Torvalds break; 5241da177e4SLinus Torvalds 525f36fe1c4SJiri Pirko tcf_block_netif_keep_dst(tp->chain->block); 5261109c005SJohn Fastabend rcu_assign_pointer(f->next, f1); 5271109c005SJohn Fastabend rcu_assign_pointer(*fp, f); 5281da177e4SLinus Torvalds 529*9ad36309SThadeu Lima de Souza Cascardo if (fold) { 5301109c005SJohn Fastabend th = to_hash(fold->handle); 5311109c005SJohn Fastabend h = from_hash(fold->handle >> 16); 5321109c005SJohn Fastabend b = rtnl_dereference(head->table[th]); 533cc7ec456SEric Dumazet if (b) { 5341109c005SJohn Fastabend fp = &b->ht[h]; 5351109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp); pfp; 5361109c005SJohn Fastabend fp = &pfp->next, pfp = rtnl_dereference(*fp)) { 537ef299cc3SCong Wang if (pfp == fold) { 538ef299cc3SCong Wang rcu_assign_pointer(*fp, fold->next); 5391da177e4SLinus Torvalds break; 5401da177e4SLinus Torvalds } 5411da177e4SLinus Torvalds } 5421da177e4SLinus Torvalds } 5431da177e4SLinus Torvalds } 5441da177e4SLinus Torvalds 5451109c005SJohn Fastabend route4_reset_fastmap(head); 5468113c095SWANG Cong *arg = f; 54718cdb37eSJohn Fastabend if (fold) { 54818cdb37eSJohn Fastabend tcf_unbind_filter(tp, &fold->res); 5493fd51de5SCong Wang tcf_exts_get_net(&fold->exts); 550aaa908ffSCong Wang tcf_queue_work(&fold->rwork, route4_delete_filter_work); 55118cdb37eSJohn Fastabend } 5521da177e4SLinus Torvalds return 0; 5531da177e4SLinus Torvalds 5541da177e4SLinus Torvalds errout: 55521641c2eSWANG Cong if (f) 556b9a24bb7SWANG Cong tcf_exts_destroy(&f->exts); 5571da177e4SLinus Torvalds kfree(f); 5581da177e4SLinus Torvalds return err; 5591da177e4SLinus Torvalds } 5601da177e4SLinus Torvalds 56112db03b6SVlad Buslov static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg, 56212db03b6SVlad Buslov bool rtnl_held) 5631da177e4SLinus Torvalds { 5641109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 565cc7ec456SEric Dumazet unsigned int h, h1; 5661da177e4SLinus Torvalds 5673027ff41SVlad Buslov if (head == NULL || arg->stop) 5681da177e4SLinus Torvalds return; 5691da177e4SLinus Torvalds 5701da177e4SLinus Torvalds for (h = 0; h <= 256; h++) { 5711109c005SJohn Fastabend struct route4_bucket *b = rtnl_dereference(head->table[h]); 5721da177e4SLinus Torvalds 5731da177e4SLinus Torvalds if (b) { 5741da177e4SLinus Torvalds for (h1 = 0; h1 <= 32; h1++) { 5751da177e4SLinus Torvalds struct route4_filter *f; 5761da177e4SLinus Torvalds 5771109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h1]); 5781109c005SJohn Fastabend f; 5791109c005SJohn Fastabend f = rtnl_dereference(f->next)) { 5801da177e4SLinus Torvalds if (arg->count < arg->skip) { 5811da177e4SLinus Torvalds arg->count++; 5821da177e4SLinus Torvalds continue; 5831da177e4SLinus Torvalds } 5848113c095SWANG Cong if (arg->fn(tp, f, arg) < 0) { 5851da177e4SLinus Torvalds arg->stop = 1; 5861da177e4SLinus Torvalds return; 5871da177e4SLinus Torvalds } 5881da177e4SLinus Torvalds arg->count++; 5891da177e4SLinus Torvalds } 5901da177e4SLinus Torvalds } 5911da177e4SLinus Torvalds } 5921da177e4SLinus Torvalds } 5931da177e4SLinus Torvalds } 5941da177e4SLinus Torvalds 5958113c095SWANG Cong static int route4_dump(struct net *net, struct tcf_proto *tp, void *fh, 59612db03b6SVlad Buslov struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 5971da177e4SLinus Torvalds { 5988113c095SWANG Cong struct route4_filter *f = fh; 5994b3550efSPatrick McHardy struct nlattr *nest; 6001da177e4SLinus Torvalds u32 id; 6011da177e4SLinus Torvalds 6021da177e4SLinus Torvalds if (f == NULL) 6031da177e4SLinus Torvalds return skb->len; 6041da177e4SLinus Torvalds 6051da177e4SLinus Torvalds t->tcm_handle = f->handle; 6061da177e4SLinus Torvalds 607ae0be8deSMichal Kubecek nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 6084b3550efSPatrick McHardy if (nest == NULL) 6094b3550efSPatrick McHardy goto nla_put_failure; 6101da177e4SLinus Torvalds 6111da177e4SLinus Torvalds if (!(f->handle & 0x8000)) { 6121da177e4SLinus Torvalds id = f->id & 0xFF; 6131b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_TO, id)) 6141b34ec43SDavid S. Miller goto nla_put_failure; 6151da177e4SLinus Torvalds } 6161da177e4SLinus Torvalds if (f->handle & 0x80000000) { 6171b34ec43SDavid S. Miller if ((f->handle >> 16) != 0xFFFF && 6181b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_IIF, f->iif)) 6191b34ec43SDavid S. Miller goto nla_put_failure; 6201da177e4SLinus Torvalds } else { 6211da177e4SLinus Torvalds id = f->id >> 16; 6221b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_FROM, id)) 6231b34ec43SDavid S. Miller goto nla_put_failure; 6241da177e4SLinus Torvalds } 6251b34ec43SDavid S. Miller if (f->res.classid && 6261b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_CLASSID, f->res.classid)) 6271b34ec43SDavid S. Miller goto nla_put_failure; 6281da177e4SLinus Torvalds 6295da57f42SWANG Cong if (tcf_exts_dump(skb, &f->exts) < 0) 630add93b61SPatrick McHardy goto nla_put_failure; 6311da177e4SLinus Torvalds 6324b3550efSPatrick McHardy nla_nest_end(skb, nest); 6331da177e4SLinus Torvalds 6345da57f42SWANG Cong if (tcf_exts_dump_stats(skb, &f->exts) < 0) 635add93b61SPatrick McHardy goto nla_put_failure; 6361da177e4SLinus Torvalds 6371da177e4SLinus Torvalds return skb->len; 6381da177e4SLinus Torvalds 639add93b61SPatrick McHardy nla_put_failure: 6406ea3b446SJiri Pirko nla_nest_cancel(skb, nest); 6411da177e4SLinus Torvalds return -1; 6421da177e4SLinus Torvalds } 6431da177e4SLinus Torvalds 6442e24cd75SCong Wang static void route4_bind_class(void *fh, u32 classid, unsigned long cl, void *q, 6452e24cd75SCong Wang unsigned long base) 64607d79fc7SCong Wang { 64707d79fc7SCong Wang struct route4_filter *f = fh; 64807d79fc7SCong Wang 6492e24cd75SCong Wang if (f && f->res.classid == classid) { 6502e24cd75SCong Wang if (cl) 6512e24cd75SCong Wang __tcf_bind_filter(q, &f->res, base); 6522e24cd75SCong Wang else 6532e24cd75SCong Wang __tcf_unbind_filter(q, &f->res); 6542e24cd75SCong Wang } 65507d79fc7SCong Wang } 65607d79fc7SCong Wang 6572eb9d75cSPatrick McHardy static struct tcf_proto_ops cls_route4_ops __read_mostly = { 6581da177e4SLinus Torvalds .kind = "route", 6591da177e4SLinus Torvalds .classify = route4_classify, 6601da177e4SLinus Torvalds .init = route4_init, 6611da177e4SLinus Torvalds .destroy = route4_destroy, 6621da177e4SLinus Torvalds .get = route4_get, 6631da177e4SLinus Torvalds .change = route4_change, 6641da177e4SLinus Torvalds .delete = route4_delete, 6651da177e4SLinus Torvalds .walk = route4_walk, 6661da177e4SLinus Torvalds .dump = route4_dump, 66707d79fc7SCong Wang .bind_class = route4_bind_class, 6681da177e4SLinus Torvalds .owner = THIS_MODULE, 6691da177e4SLinus Torvalds }; 6701da177e4SLinus Torvalds 6711da177e4SLinus Torvalds static int __init init_route4(void) 6721da177e4SLinus Torvalds { 6731da177e4SLinus Torvalds return register_tcf_proto_ops(&cls_route4_ops); 6741da177e4SLinus Torvalds } 6751da177e4SLinus Torvalds 6761da177e4SLinus Torvalds static void __exit exit_route4(void) 6771da177e4SLinus Torvalds { 6781da177e4SLinus Torvalds unregister_tcf_proto_ops(&cls_route4_ops); 6791da177e4SLinus Torvalds } 6801da177e4SLinus Torvalds 6811da177e4SLinus Torvalds module_init(init_route4) 6821da177e4SLinus Torvalds module_exit(exit_route4) 6831da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 684