11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * net/sched/cls_route.c ROUTE4 classifier. 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * This program is free software; you can redistribute it and/or 51da177e4SLinus Torvalds * modify it under the terms of the GNU General Public License 61da177e4SLinus Torvalds * as published by the Free Software Foundation; either version 71da177e4SLinus Torvalds * 2 of the License, or (at your option) any later version. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 121da177e4SLinus Torvalds #include <linux/module.h> 135a0e3ad6STejun Heo #include <linux/slab.h> 141da177e4SLinus Torvalds #include <linux/types.h> 151da177e4SLinus Torvalds #include <linux/kernel.h> 161da177e4SLinus Torvalds #include <linux/string.h> 171da177e4SLinus Torvalds #include <linux/errno.h> 181da177e4SLinus Torvalds #include <linux/skbuff.h> 190ba48053SPatrick McHardy #include <net/dst.h> 200ba48053SPatrick McHardy #include <net/route.h> 210ba48053SPatrick McHardy #include <net/netlink.h> 221da177e4SLinus Torvalds #include <net/act_api.h> 231da177e4SLinus Torvalds #include <net/pkt_cls.h> 241da177e4SLinus Torvalds 251da177e4SLinus Torvalds /* 26cc7ec456SEric Dumazet * 1. For now we assume that route tags < 256. 27cc7ec456SEric Dumazet * It allows to use direct table lookups, instead of hash tables. 28cc7ec456SEric Dumazet * 2. For now we assume that "from TAG" and "fromdev DEV" statements 29cc7ec456SEric Dumazet * are mutually exclusive. 30cc7ec456SEric Dumazet * 3. "to TAG from ANY" has higher priority, than "to ANY from XXX" 311da177e4SLinus Torvalds */ 32cc7ec456SEric Dumazet struct route4_fastmap { 331da177e4SLinus Torvalds struct route4_filter *filter; 341da177e4SLinus Torvalds u32 id; 351da177e4SLinus Torvalds int iif; 361da177e4SLinus Torvalds }; 371da177e4SLinus Torvalds 38cc7ec456SEric Dumazet struct route4_head { 391da177e4SLinus Torvalds struct route4_fastmap fastmap[16]; 401109c005SJohn Fastabend struct route4_bucket __rcu *table[256 + 1]; 411109c005SJohn Fastabend struct rcu_head rcu; 421da177e4SLinus Torvalds }; 431da177e4SLinus Torvalds 44cc7ec456SEric Dumazet struct route4_bucket { 451da177e4SLinus Torvalds /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */ 461109c005SJohn Fastabend struct route4_filter __rcu *ht[16 + 16 + 1]; 471109c005SJohn Fastabend struct rcu_head rcu; 481da177e4SLinus Torvalds }; 491da177e4SLinus Torvalds 50cc7ec456SEric Dumazet struct route4_filter { 511109c005SJohn Fastabend struct route4_filter __rcu *next; 521da177e4SLinus Torvalds u32 id; 531da177e4SLinus Torvalds int iif; 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds struct tcf_result res; 561da177e4SLinus Torvalds struct tcf_exts exts; 571da177e4SLinus Torvalds u32 handle; 581da177e4SLinus Torvalds struct route4_bucket *bkt; 591109c005SJohn Fastabend struct tcf_proto *tp; 60c2f3f31dSCong Wang union { 61c2f3f31dSCong Wang struct work_struct work; 621109c005SJohn Fastabend struct rcu_head rcu; 631da177e4SLinus Torvalds }; 64c2f3f31dSCong Wang }; 651da177e4SLinus Torvalds 661da177e4SLinus Torvalds #define ROUTE4_FAILURE ((struct route4_filter *)(-1L)) 671da177e4SLinus Torvalds 68cc7ec456SEric Dumazet static inline int route4_fastmap_hash(u32 id, int iif) 691da177e4SLinus Torvalds { 701da177e4SLinus Torvalds return id & 0xF; 711da177e4SLinus Torvalds } 721da177e4SLinus Torvalds 731109c005SJohn Fastabend static DEFINE_SPINLOCK(fastmap_lock); 74cc7ec456SEric Dumazet static void 751109c005SJohn Fastabend route4_reset_fastmap(struct route4_head *head) 761da177e4SLinus Torvalds { 771109c005SJohn Fastabend spin_lock_bh(&fastmap_lock); 781da177e4SLinus Torvalds memset(head->fastmap, 0, sizeof(head->fastmap)); 791109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock); 801da177e4SLinus Torvalds } 811da177e4SLinus Torvalds 82cc7ec456SEric Dumazet static void 831da177e4SLinus Torvalds route4_set_fastmap(struct route4_head *head, u32 id, int iif, 841da177e4SLinus Torvalds struct route4_filter *f) 851da177e4SLinus Torvalds { 861da177e4SLinus Torvalds int h = route4_fastmap_hash(id, iif); 87cc7ec456SEric Dumazet 881109c005SJohn Fastabend /* fastmap updates must look atomic to aling id, iff, filter */ 891109c005SJohn Fastabend spin_lock_bh(&fastmap_lock); 901da177e4SLinus Torvalds head->fastmap[h].id = id; 911da177e4SLinus Torvalds head->fastmap[h].iif = iif; 921da177e4SLinus Torvalds head->fastmap[h].filter = f; 931109c005SJohn Fastabend spin_unlock_bh(&fastmap_lock); 941da177e4SLinus Torvalds } 951da177e4SLinus Torvalds 96cc7ec456SEric Dumazet static inline int route4_hash_to(u32 id) 971da177e4SLinus Torvalds { 981da177e4SLinus Torvalds return id & 0xFF; 991da177e4SLinus Torvalds } 1001da177e4SLinus Torvalds 101cc7ec456SEric Dumazet static inline int route4_hash_from(u32 id) 1021da177e4SLinus Torvalds { 1031da177e4SLinus Torvalds return (id >> 16) & 0xF; 1041da177e4SLinus Torvalds } 1051da177e4SLinus Torvalds 106cc7ec456SEric Dumazet static inline int route4_hash_iif(int iif) 1071da177e4SLinus Torvalds { 1081da177e4SLinus Torvalds return 16 + ((iif >> 16) & 0xF); 1091da177e4SLinus Torvalds } 1101da177e4SLinus Torvalds 111cc7ec456SEric Dumazet static inline int route4_hash_wild(void) 1121da177e4SLinus Torvalds { 1131da177e4SLinus Torvalds return 32; 1141da177e4SLinus Torvalds } 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds #define ROUTE4_APPLY_RESULT() \ 1171da177e4SLinus Torvalds { \ 1181da177e4SLinus Torvalds *res = f->res; \ 1196fc6d06eSJiri Pirko if (tcf_exts_has_actions(&f->exts)) { \ 1201da177e4SLinus Torvalds int r = tcf_exts_exec(skb, &f->exts, res); \ 1211da177e4SLinus Torvalds if (r < 0) { \ 1221da177e4SLinus Torvalds dont_cache = 1; \ 1231da177e4SLinus Torvalds continue; \ 1241da177e4SLinus Torvalds } \ 1251da177e4SLinus Torvalds return r; \ 1261da177e4SLinus Torvalds } else if (!dont_cache) \ 1271da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, f); \ 1281da177e4SLinus Torvalds return 0; \ 1291da177e4SLinus Torvalds } 1301da177e4SLinus Torvalds 131dc7f9f6eSEric Dumazet static int route4_classify(struct sk_buff *skb, const struct tcf_proto *tp, 1321da177e4SLinus Torvalds struct tcf_result *res) 1331da177e4SLinus Torvalds { 1341109c005SJohn Fastabend struct route4_head *head = rcu_dereference_bh(tp->root); 1351da177e4SLinus Torvalds struct dst_entry *dst; 1361da177e4SLinus Torvalds struct route4_bucket *b; 1371da177e4SLinus Torvalds struct route4_filter *f; 1381da177e4SLinus Torvalds u32 id, h; 1391da177e4SLinus Torvalds int iif, dont_cache = 0; 1401da177e4SLinus Torvalds 141cc7ec456SEric Dumazet dst = skb_dst(skb); 142cc7ec456SEric Dumazet if (!dst) 1431da177e4SLinus Torvalds goto failure; 1441da177e4SLinus Torvalds 1451da177e4SLinus Torvalds id = dst->tclassid; 1461da177e4SLinus Torvalds 14792101b3bSDavid S. Miller iif = inet_iif(skb); 1481da177e4SLinus Torvalds 1491da177e4SLinus Torvalds h = route4_fastmap_hash(id, iif); 1501109c005SJohn Fastabend 1511109c005SJohn Fastabend spin_lock(&fastmap_lock); 1521da177e4SLinus Torvalds if (id == head->fastmap[h].id && 1531da177e4SLinus Torvalds iif == head->fastmap[h].iif && 1541da177e4SLinus Torvalds (f = head->fastmap[h].filter) != NULL) { 1551109c005SJohn Fastabend if (f == ROUTE4_FAILURE) { 1561109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1571da177e4SLinus Torvalds goto failure; 1581109c005SJohn Fastabend } 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds *res = f->res; 1611109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1621da177e4SLinus Torvalds return 0; 1631da177e4SLinus Torvalds } 1641109c005SJohn Fastabend spin_unlock(&fastmap_lock); 1651da177e4SLinus Torvalds 1661da177e4SLinus Torvalds h = route4_hash_to(id); 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds restart: 1691109c005SJohn Fastabend b = rcu_dereference_bh(head->table[h]); 170cc7ec456SEric Dumazet if (b) { 1711109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]); 1721109c005SJohn Fastabend f; 1731109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1741da177e4SLinus Torvalds if (f->id == id) 1751da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1761da177e4SLinus Torvalds 1771109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]); 1781109c005SJohn Fastabend f; 1791109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1801da177e4SLinus Torvalds if (f->iif == iif) 1811da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1821da177e4SLinus Torvalds 1831109c005SJohn Fastabend for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]); 1841109c005SJohn Fastabend f; 1851109c005SJohn Fastabend f = rcu_dereference_bh(f->next)) 1861da177e4SLinus Torvalds ROUTE4_APPLY_RESULT(); 1871da177e4SLinus Torvalds } 1881da177e4SLinus Torvalds if (h < 256) { 1891da177e4SLinus Torvalds h = 256; 1901da177e4SLinus Torvalds id &= ~0xFFFF; 1911da177e4SLinus Torvalds goto restart; 1921da177e4SLinus Torvalds } 1931da177e4SLinus Torvalds 1941da177e4SLinus Torvalds if (!dont_cache) 1951da177e4SLinus Torvalds route4_set_fastmap(head, id, iif, ROUTE4_FAILURE); 1961da177e4SLinus Torvalds failure: 1971da177e4SLinus Torvalds return -1; 1981da177e4SLinus Torvalds } 1991da177e4SLinus Torvalds 2001da177e4SLinus Torvalds static inline u32 to_hash(u32 id) 2011da177e4SLinus Torvalds { 2021da177e4SLinus Torvalds u32 h = id & 0xFF; 203cc7ec456SEric Dumazet 2041da177e4SLinus Torvalds if (id & 0x8000) 2051da177e4SLinus Torvalds h += 256; 2061da177e4SLinus Torvalds return h; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 2091da177e4SLinus Torvalds static inline u32 from_hash(u32 id) 2101da177e4SLinus Torvalds { 2111da177e4SLinus Torvalds id &= 0xFFFF; 2121da177e4SLinus Torvalds if (id == 0xFFFF) 2131da177e4SLinus Torvalds return 32; 2141da177e4SLinus Torvalds if (!(id & 0x8000)) { 2151da177e4SLinus Torvalds if (id > 255) 2161da177e4SLinus Torvalds return 256; 2171da177e4SLinus Torvalds return id & 0xF; 2181da177e4SLinus Torvalds } 2191da177e4SLinus Torvalds return 16 + (id & 0xF); 2201da177e4SLinus Torvalds } 2211da177e4SLinus Torvalds 2228113c095SWANG Cong static void *route4_get(struct tcf_proto *tp, u32 handle) 2231da177e4SLinus Torvalds { 2241109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 2251da177e4SLinus Torvalds struct route4_bucket *b; 2261da177e4SLinus Torvalds struct route4_filter *f; 227cc7ec456SEric Dumazet unsigned int h1, h2; 2281da177e4SLinus Torvalds 2291da177e4SLinus Torvalds h1 = to_hash(handle); 2301da177e4SLinus Torvalds if (h1 > 256) 2318113c095SWANG Cong return NULL; 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds h2 = from_hash(handle >> 16); 2341da177e4SLinus Torvalds if (h2 > 32) 2358113c095SWANG Cong return NULL; 2361da177e4SLinus Torvalds 2371109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 238cc7ec456SEric Dumazet if (b) { 2391109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h2]); 2401109c005SJohn Fastabend f; 2411109c005SJohn Fastabend f = rtnl_dereference(f->next)) 2421da177e4SLinus Torvalds if (f->handle == handle) 2438113c095SWANG Cong return f; 2441da177e4SLinus Torvalds } 2458113c095SWANG Cong return NULL; 2461da177e4SLinus Torvalds } 2471da177e4SLinus Torvalds 2481da177e4SLinus Torvalds static int route4_init(struct tcf_proto *tp) 2491da177e4SLinus Torvalds { 250a05c2d11SWANG Cong struct route4_head *head; 251a05c2d11SWANG Cong 252a05c2d11SWANG Cong head = kzalloc(sizeof(struct route4_head), GFP_KERNEL); 253a05c2d11SWANG Cong if (head == NULL) 254a05c2d11SWANG Cong return -ENOBUFS; 255a05c2d11SWANG Cong 256a05c2d11SWANG Cong rcu_assign_pointer(tp->root, head); 2571da177e4SLinus Torvalds return 0; 2581da177e4SLinus Torvalds } 2591da177e4SLinus Torvalds 2603fd51de5SCong Wang static void __route4_delete_filter(struct route4_filter *f) 2613fd51de5SCong Wang { 2623fd51de5SCong Wang tcf_exts_destroy(&f->exts); 2633fd51de5SCong Wang tcf_exts_put_net(&f->exts); 2643fd51de5SCong Wang kfree(f); 2653fd51de5SCong Wang } 2663fd51de5SCong Wang 267c2f3f31dSCong Wang static void route4_delete_filter_work(struct work_struct *work) 268c2f3f31dSCong Wang { 269c2f3f31dSCong Wang struct route4_filter *f = container_of(work, struct route4_filter, work); 270c2f3f31dSCong Wang 271c2f3f31dSCong Wang rtnl_lock(); 2723fd51de5SCong Wang __route4_delete_filter(f); 273c2f3f31dSCong Wang rtnl_unlock(); 274c2f3f31dSCong Wang } 275c2f3f31dSCong Wang 2765a7a5555SJamal Hadi Salim static void route4_delete_filter(struct rcu_head *head) 2771da177e4SLinus Torvalds { 2781109c005SJohn Fastabend struct route4_filter *f = container_of(head, struct route4_filter, rcu); 2791109c005SJohn Fastabend 280c2f3f31dSCong Wang INIT_WORK(&f->work, route4_delete_filter_work); 281c2f3f31dSCong Wang tcf_queue_work(&f->work); 2821da177e4SLinus Torvalds } 2831da177e4SLinus Torvalds 284763dbf63SWANG Cong static void route4_destroy(struct tcf_proto *tp) 2851da177e4SLinus Torvalds { 2861109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 2871da177e4SLinus Torvalds int h1, h2; 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds if (head == NULL) 290763dbf63SWANG Cong return; 2911da177e4SLinus Torvalds 2921da177e4SLinus Torvalds for (h1 = 0; h1 <= 256; h1++) { 2931da177e4SLinus Torvalds struct route4_bucket *b; 2941da177e4SLinus Torvalds 2951109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 296cc7ec456SEric Dumazet if (b) { 2971da177e4SLinus Torvalds for (h2 = 0; h2 <= 32; h2++) { 2981da177e4SLinus Torvalds struct route4_filter *f; 2991da177e4SLinus Torvalds 3001109c005SJohn Fastabend while ((f = rtnl_dereference(b->ht[h2])) != NULL) { 3011109c005SJohn Fastabend struct route4_filter *next; 3021109c005SJohn Fastabend 3031109c005SJohn Fastabend next = rtnl_dereference(f->next); 3041109c005SJohn Fastabend RCU_INIT_POINTER(b->ht[h2], next); 30518cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res); 3063fd51de5SCong Wang if (tcf_exts_get_net(&f->exts)) 3071109c005SJohn Fastabend call_rcu(&f->rcu, route4_delete_filter); 3083fd51de5SCong Wang else 3093fd51de5SCong Wang __route4_delete_filter(f); 3101da177e4SLinus Torvalds } 3111da177e4SLinus Torvalds } 3121109c005SJohn Fastabend RCU_INIT_POINTER(head->table[h1], NULL); 3131109c005SJohn Fastabend kfree_rcu(b, rcu); 3141da177e4SLinus Torvalds } 3151da177e4SLinus Torvalds } 3161109c005SJohn Fastabend kfree_rcu(head, rcu); 3171da177e4SLinus Torvalds } 3181da177e4SLinus Torvalds 319571acf21SAlexander Aring static int route4_delete(struct tcf_proto *tp, void *arg, bool *last, 320571acf21SAlexander Aring struct netlink_ext_ack *extack) 3211da177e4SLinus Torvalds { 3221109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 3238113c095SWANG Cong struct route4_filter *f = arg; 3241109c005SJohn Fastabend struct route4_filter __rcu **fp; 3251109c005SJohn Fastabend struct route4_filter *nf; 3261da177e4SLinus Torvalds struct route4_bucket *b; 3271109c005SJohn Fastabend unsigned int h = 0; 328763dbf63SWANG Cong int i, h1; 3291da177e4SLinus Torvalds 3301da177e4SLinus Torvalds if (!head || !f) 3311da177e4SLinus Torvalds return -EINVAL; 3321da177e4SLinus Torvalds 3331da177e4SLinus Torvalds h = f->handle; 3341da177e4SLinus Torvalds b = f->bkt; 3351da177e4SLinus Torvalds 3361109c005SJohn Fastabend fp = &b->ht[from_hash(h >> 16)]; 3371109c005SJohn Fastabend for (nf = rtnl_dereference(*fp); nf; 3381109c005SJohn Fastabend fp = &nf->next, nf = rtnl_dereference(*fp)) { 3391109c005SJohn Fastabend if (nf == f) { 3401109c005SJohn Fastabend /* unlink it */ 3411109c005SJohn Fastabend RCU_INIT_POINTER(*fp, rtnl_dereference(f->next)); 3421da177e4SLinus Torvalds 3431109c005SJohn Fastabend /* Remove any fastmap lookups that might ref filter 3441109c005SJohn Fastabend * notice we unlink'd the filter so we can't get it 3451109c005SJohn Fastabend * back in the fastmap. 3461109c005SJohn Fastabend */ 3471109c005SJohn Fastabend route4_reset_fastmap(head); 3481da177e4SLinus Torvalds 3491109c005SJohn Fastabend /* Delete it */ 35018cdb37eSJohn Fastabend tcf_unbind_filter(tp, &f->res); 3513fd51de5SCong Wang tcf_exts_get_net(&f->exts); 3521109c005SJohn Fastabend call_rcu(&f->rcu, route4_delete_filter); 3531da177e4SLinus Torvalds 3541109c005SJohn Fastabend /* Strip RTNL protected tree */ 3551109c005SJohn Fastabend for (i = 0; i <= 32; i++) { 3561109c005SJohn Fastabend struct route4_filter *rt; 3571109c005SJohn Fastabend 3581109c005SJohn Fastabend rt = rtnl_dereference(b->ht[i]); 3591109c005SJohn Fastabend if (rt) 360763dbf63SWANG Cong goto out; 3611109c005SJohn Fastabend } 3621da177e4SLinus Torvalds 3631da177e4SLinus Torvalds /* OK, session has no flows */ 3641109c005SJohn Fastabend RCU_INIT_POINTER(head->table[to_hash(h)], NULL); 3651109c005SJohn Fastabend kfree_rcu(b, rcu); 366763dbf63SWANG Cong break; 367763dbf63SWANG Cong } 368763dbf63SWANG Cong } 3691da177e4SLinus Torvalds 370763dbf63SWANG Cong out: 371763dbf63SWANG Cong *last = true; 372763dbf63SWANG Cong for (h1 = 0; h1 <= 256; h1++) { 373763dbf63SWANG Cong if (rcu_access_pointer(head->table[h1])) { 374763dbf63SWANG Cong *last = false; 375763dbf63SWANG Cong break; 3761da177e4SLinus Torvalds } 3771da177e4SLinus Torvalds } 378763dbf63SWANG Cong 3791da177e4SLinus Torvalds return 0; 3801da177e4SLinus Torvalds } 3811da177e4SLinus Torvalds 3826fa8c014SPatrick McHardy static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = { 3836fa8c014SPatrick McHardy [TCA_ROUTE4_CLASSID] = { .type = NLA_U32 }, 3846fa8c014SPatrick McHardy [TCA_ROUTE4_TO] = { .type = NLA_U32 }, 3856fa8c014SPatrick McHardy [TCA_ROUTE4_FROM] = { .type = NLA_U32 }, 3866fa8c014SPatrick McHardy [TCA_ROUTE4_IIF] = { .type = NLA_U32 }, 3876fa8c014SPatrick McHardy }; 3886fa8c014SPatrick McHardy 389c1b52739SBenjamin LaHaise static int route4_set_parms(struct net *net, struct tcf_proto *tp, 390c1b52739SBenjamin LaHaise unsigned long base, struct route4_filter *f, 391c1b52739SBenjamin LaHaise u32 handle, struct route4_head *head, 3922f7ef2f8SCong Wang struct nlattr **tb, struct nlattr *est, int new, 39350a56190SAlexander Aring bool ovr, struct netlink_ext_ack *extack) 3941da177e4SLinus Torvalds { 3951da177e4SLinus Torvalds u32 id = 0, to = 0, nhandle = 0x8000; 3961da177e4SLinus Torvalds struct route4_filter *fp; 3971da177e4SLinus Torvalds unsigned int h1; 3981da177e4SLinus Torvalds struct route4_bucket *b; 399b9a24bb7SWANG Cong int err; 4001da177e4SLinus Torvalds 40150a56190SAlexander Aring err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, extack); 4021da177e4SLinus Torvalds if (err < 0) 4031da177e4SLinus Torvalds return err; 4041da177e4SLinus Torvalds 405add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO]) { 4061da177e4SLinus Torvalds if (new && handle & 0x8000) 4078c98d571SJiri Pirko return -EINVAL; 4081587bac4SPatrick McHardy to = nla_get_u32(tb[TCA_ROUTE4_TO]); 4091da177e4SLinus Torvalds if (to > 0xFF) 4108c98d571SJiri Pirko return -EINVAL; 4111da177e4SLinus Torvalds nhandle = to; 4121da177e4SLinus Torvalds } 4131da177e4SLinus Torvalds 414add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM]) { 415add93b61SPatrick McHardy if (tb[TCA_ROUTE4_IIF]) 4168c98d571SJiri Pirko return -EINVAL; 4171587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_FROM]); 4181da177e4SLinus Torvalds if (id > 0xFF) 4198c98d571SJiri Pirko return -EINVAL; 4201da177e4SLinus Torvalds nhandle |= id << 16; 421add93b61SPatrick McHardy } else if (tb[TCA_ROUTE4_IIF]) { 4221587bac4SPatrick McHardy id = nla_get_u32(tb[TCA_ROUTE4_IIF]); 4231da177e4SLinus Torvalds if (id > 0x7FFF) 4248c98d571SJiri Pirko return -EINVAL; 4251da177e4SLinus Torvalds nhandle |= (id | 0x8000) << 16; 4261da177e4SLinus Torvalds } else 4271da177e4SLinus Torvalds nhandle |= 0xFFFF << 16; 4281da177e4SLinus Torvalds 4291da177e4SLinus Torvalds if (handle && new) { 4301da177e4SLinus Torvalds nhandle |= handle & 0x7F00; 4311da177e4SLinus Torvalds if (nhandle != handle) 4328c98d571SJiri Pirko return -EINVAL; 4331da177e4SLinus Torvalds } 4341da177e4SLinus Torvalds 4351da177e4SLinus Torvalds h1 = to_hash(nhandle); 4361109c005SJohn Fastabend b = rtnl_dereference(head->table[h1]); 437cc7ec456SEric Dumazet if (!b) { 4380da974f4SPanagiotis Issaris b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL); 4391da177e4SLinus Torvalds if (b == NULL) 4408c98d571SJiri Pirko return -ENOBUFS; 4411da177e4SLinus Torvalds 4421109c005SJohn Fastabend rcu_assign_pointer(head->table[h1], b); 4431da177e4SLinus Torvalds } else { 4441da177e4SLinus Torvalds unsigned int h2 = from_hash(nhandle >> 16); 445cc7ec456SEric Dumazet 4461109c005SJohn Fastabend for (fp = rtnl_dereference(b->ht[h2]); 4471109c005SJohn Fastabend fp; 4481109c005SJohn Fastabend fp = rtnl_dereference(fp->next)) 4491da177e4SLinus Torvalds if (fp->handle == f->handle) 4508c98d571SJiri Pirko return -EEXIST; 4511da177e4SLinus Torvalds } 4521da177e4SLinus Torvalds 453add93b61SPatrick McHardy if (tb[TCA_ROUTE4_TO]) 4541da177e4SLinus Torvalds f->id = to; 4551da177e4SLinus Torvalds 456add93b61SPatrick McHardy if (tb[TCA_ROUTE4_FROM]) 4571da177e4SLinus Torvalds f->id = to | id<<16; 458add93b61SPatrick McHardy else if (tb[TCA_ROUTE4_IIF]) 4591da177e4SLinus Torvalds f->iif = id; 4601da177e4SLinus Torvalds 4611da177e4SLinus Torvalds f->handle = nhandle; 4621da177e4SLinus Torvalds f->bkt = b; 4631109c005SJohn Fastabend f->tp = tp; 4641da177e4SLinus Torvalds 465add93b61SPatrick McHardy if (tb[TCA_ROUTE4_CLASSID]) { 4661587bac4SPatrick McHardy f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]); 4671da177e4SLinus Torvalds tcf_bind_filter(tp, &f->res, base); 4681da177e4SLinus Torvalds } 4691da177e4SLinus Torvalds 4701da177e4SLinus Torvalds return 0; 4711da177e4SLinus Torvalds } 4721da177e4SLinus Torvalds 473c1b52739SBenjamin LaHaise static int route4_change(struct net *net, struct sk_buff *in_skb, 4745a7a5555SJamal Hadi Salim struct tcf_proto *tp, unsigned long base, u32 handle, 4757306db38SAlexander Aring struct nlattr **tca, void **arg, bool ovr, 4767306db38SAlexander Aring struct netlink_ext_ack *extack) 4771da177e4SLinus Torvalds { 4781109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 4791109c005SJohn Fastabend struct route4_filter __rcu **fp; 4801109c005SJohn Fastabend struct route4_filter *fold, *f1, *pfp, *f = NULL; 4811da177e4SLinus Torvalds struct route4_bucket *b; 482add93b61SPatrick McHardy struct nlattr *opt = tca[TCA_OPTIONS]; 483add93b61SPatrick McHardy struct nlattr *tb[TCA_ROUTE4_MAX + 1]; 4841da177e4SLinus Torvalds unsigned int h, th; 4851da177e4SLinus Torvalds int err; 4861109c005SJohn Fastabend bool new = true; 4871da177e4SLinus Torvalds 4881da177e4SLinus Torvalds if (opt == NULL) 4891da177e4SLinus Torvalds return handle ? -EINVAL : 0; 4901da177e4SLinus Torvalds 491fceb6435SJohannes Berg err = nla_parse_nested(tb, TCA_ROUTE4_MAX, opt, route4_policy, NULL); 492cee63723SPatrick McHardy if (err < 0) 493cee63723SPatrick McHardy return err; 4941da177e4SLinus Torvalds 4958113c095SWANG Cong fold = *arg; 4961109c005SJohn Fastabend if (fold && handle && fold->handle != handle) 4971da177e4SLinus Torvalds return -EINVAL; 4981da177e4SLinus Torvalds 4991da177e4SLinus Torvalds err = -ENOBUFS; 5000da974f4SPanagiotis Issaris f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL); 5011109c005SJohn Fastabend if (!f) 5021da177e4SLinus Torvalds goto errout; 5031da177e4SLinus Torvalds 504b9a24bb7SWANG Cong err = tcf_exts_init(&f->exts, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE); 505b9a24bb7SWANG Cong if (err < 0) 506b9a24bb7SWANG Cong goto errout; 507b9a24bb7SWANG Cong 5081109c005SJohn Fastabend if (fold) { 5091109c005SJohn Fastabend f->id = fold->id; 5101109c005SJohn Fastabend f->iif = fold->iif; 5111109c005SJohn Fastabend f->res = fold->res; 5121109c005SJohn Fastabend f->handle = fold->handle; 5131109c005SJohn Fastabend 5141109c005SJohn Fastabend f->tp = fold->tp; 5151109c005SJohn Fastabend f->bkt = fold->bkt; 5161109c005SJohn Fastabend new = false; 5171109c005SJohn Fastabend } 5181109c005SJohn Fastabend 519c1b52739SBenjamin LaHaise err = route4_set_parms(net, tp, base, f, handle, head, tb, 52050a56190SAlexander Aring tca[TCA_RATE], new, ovr, extack); 5211da177e4SLinus Torvalds if (err < 0) 5221da177e4SLinus Torvalds goto errout; 5231da177e4SLinus Torvalds 5241da177e4SLinus Torvalds h = from_hash(f->handle >> 16); 5251109c005SJohn Fastabend fp = &f->bkt->ht[h]; 5261109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp); 5271109c005SJohn Fastabend (f1 = rtnl_dereference(*fp)) != NULL; 5281109c005SJohn Fastabend fp = &f1->next) 5291da177e4SLinus Torvalds if (f->handle < f1->handle) 5301da177e4SLinus Torvalds break; 5311da177e4SLinus Torvalds 532f36fe1c4SJiri Pirko tcf_block_netif_keep_dst(tp->chain->block); 5331109c005SJohn Fastabend rcu_assign_pointer(f->next, f1); 5341109c005SJohn Fastabend rcu_assign_pointer(*fp, f); 5351da177e4SLinus Torvalds 5361109c005SJohn Fastabend if (fold && fold->handle && f->handle != fold->handle) { 5371109c005SJohn Fastabend th = to_hash(fold->handle); 5381109c005SJohn Fastabend h = from_hash(fold->handle >> 16); 5391109c005SJohn Fastabend b = rtnl_dereference(head->table[th]); 540cc7ec456SEric Dumazet if (b) { 5411109c005SJohn Fastabend fp = &b->ht[h]; 5421109c005SJohn Fastabend for (pfp = rtnl_dereference(*fp); pfp; 5431109c005SJohn Fastabend fp = &pfp->next, pfp = rtnl_dereference(*fp)) { 5441109c005SJohn Fastabend if (pfp == f) { 5451da177e4SLinus Torvalds *fp = f->next; 5461da177e4SLinus Torvalds break; 5471da177e4SLinus Torvalds } 5481da177e4SLinus Torvalds } 5491da177e4SLinus Torvalds } 5501da177e4SLinus Torvalds } 5511da177e4SLinus Torvalds 5521109c005SJohn Fastabend route4_reset_fastmap(head); 5538113c095SWANG Cong *arg = f; 55418cdb37eSJohn Fastabend if (fold) { 55518cdb37eSJohn Fastabend tcf_unbind_filter(tp, &fold->res); 5563fd51de5SCong Wang tcf_exts_get_net(&fold->exts); 5571109c005SJohn Fastabend call_rcu(&fold->rcu, route4_delete_filter); 55818cdb37eSJohn Fastabend } 5591da177e4SLinus Torvalds return 0; 5601da177e4SLinus Torvalds 5611da177e4SLinus Torvalds errout: 56221641c2eSWANG Cong if (f) 563b9a24bb7SWANG Cong tcf_exts_destroy(&f->exts); 5641da177e4SLinus Torvalds kfree(f); 5651da177e4SLinus Torvalds return err; 5661da177e4SLinus Torvalds } 5671da177e4SLinus Torvalds 5681da177e4SLinus Torvalds static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg) 5691da177e4SLinus Torvalds { 5701109c005SJohn Fastabend struct route4_head *head = rtnl_dereference(tp->root); 571cc7ec456SEric Dumazet unsigned int h, h1; 5721da177e4SLinus Torvalds 5731da177e4SLinus Torvalds if (head == NULL) 5741da177e4SLinus Torvalds arg->stop = 1; 5751da177e4SLinus Torvalds 5761da177e4SLinus Torvalds if (arg->stop) 5771da177e4SLinus Torvalds return; 5781da177e4SLinus Torvalds 5791da177e4SLinus Torvalds for (h = 0; h <= 256; h++) { 5801109c005SJohn Fastabend struct route4_bucket *b = rtnl_dereference(head->table[h]); 5811da177e4SLinus Torvalds 5821da177e4SLinus Torvalds if (b) { 5831da177e4SLinus Torvalds for (h1 = 0; h1 <= 32; h1++) { 5841da177e4SLinus Torvalds struct route4_filter *f; 5851da177e4SLinus Torvalds 5861109c005SJohn Fastabend for (f = rtnl_dereference(b->ht[h1]); 5871109c005SJohn Fastabend f; 5881109c005SJohn Fastabend f = rtnl_dereference(f->next)) { 5891da177e4SLinus Torvalds if (arg->count < arg->skip) { 5901da177e4SLinus Torvalds arg->count++; 5911da177e4SLinus Torvalds continue; 5921da177e4SLinus Torvalds } 5938113c095SWANG Cong if (arg->fn(tp, f, arg) < 0) { 5941da177e4SLinus Torvalds arg->stop = 1; 5951da177e4SLinus Torvalds return; 5961da177e4SLinus Torvalds } 5971da177e4SLinus Torvalds arg->count++; 5981da177e4SLinus Torvalds } 5991da177e4SLinus Torvalds } 6001da177e4SLinus Torvalds } 6011da177e4SLinus Torvalds } 6021da177e4SLinus Torvalds } 6031da177e4SLinus Torvalds 6048113c095SWANG Cong static int route4_dump(struct net *net, struct tcf_proto *tp, void *fh, 6051da177e4SLinus Torvalds struct sk_buff *skb, struct tcmsg *t) 6061da177e4SLinus Torvalds { 6078113c095SWANG Cong struct route4_filter *f = fh; 6084b3550efSPatrick McHardy struct nlattr *nest; 6091da177e4SLinus Torvalds u32 id; 6101da177e4SLinus Torvalds 6111da177e4SLinus Torvalds if (f == NULL) 6121da177e4SLinus Torvalds return skb->len; 6131da177e4SLinus Torvalds 6141da177e4SLinus Torvalds t->tcm_handle = f->handle; 6151da177e4SLinus Torvalds 6164b3550efSPatrick McHardy nest = nla_nest_start(skb, TCA_OPTIONS); 6174b3550efSPatrick McHardy if (nest == NULL) 6184b3550efSPatrick McHardy goto nla_put_failure; 6191da177e4SLinus Torvalds 6201da177e4SLinus Torvalds if (!(f->handle & 0x8000)) { 6211da177e4SLinus Torvalds id = f->id & 0xFF; 6221b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_TO, id)) 6231b34ec43SDavid S. Miller goto nla_put_failure; 6241da177e4SLinus Torvalds } 6251da177e4SLinus Torvalds if (f->handle & 0x80000000) { 6261b34ec43SDavid S. Miller if ((f->handle >> 16) != 0xFFFF && 6271b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_IIF, f->iif)) 6281b34ec43SDavid S. Miller goto nla_put_failure; 6291da177e4SLinus Torvalds } else { 6301da177e4SLinus Torvalds id = f->id >> 16; 6311b34ec43SDavid S. Miller if (nla_put_u32(skb, TCA_ROUTE4_FROM, id)) 6321b34ec43SDavid S. Miller goto nla_put_failure; 6331da177e4SLinus Torvalds } 6341b34ec43SDavid S. Miller if (f->res.classid && 6351b34ec43SDavid S. Miller nla_put_u32(skb, TCA_ROUTE4_CLASSID, f->res.classid)) 6361b34ec43SDavid S. Miller goto nla_put_failure; 6371da177e4SLinus Torvalds 6385da57f42SWANG Cong if (tcf_exts_dump(skb, &f->exts) < 0) 639add93b61SPatrick McHardy goto nla_put_failure; 6401da177e4SLinus Torvalds 6414b3550efSPatrick McHardy nla_nest_end(skb, nest); 6421da177e4SLinus Torvalds 6435da57f42SWANG Cong if (tcf_exts_dump_stats(skb, &f->exts) < 0) 644add93b61SPatrick McHardy goto nla_put_failure; 6451da177e4SLinus Torvalds 6461da177e4SLinus Torvalds return skb->len; 6471da177e4SLinus Torvalds 648add93b61SPatrick McHardy nla_put_failure: 6496ea3b446SJiri Pirko nla_nest_cancel(skb, nest); 6501da177e4SLinus Torvalds return -1; 6511da177e4SLinus Torvalds } 6521da177e4SLinus Torvalds 65307d79fc7SCong Wang static void route4_bind_class(void *fh, u32 classid, unsigned long cl) 65407d79fc7SCong Wang { 65507d79fc7SCong Wang struct route4_filter *f = fh; 65607d79fc7SCong Wang 65707d79fc7SCong Wang if (f && f->res.classid == classid) 65807d79fc7SCong Wang f->res.class = cl; 65907d79fc7SCong Wang } 66007d79fc7SCong Wang 6612eb9d75cSPatrick McHardy static struct tcf_proto_ops cls_route4_ops __read_mostly = { 6621da177e4SLinus Torvalds .kind = "route", 6631da177e4SLinus Torvalds .classify = route4_classify, 6641da177e4SLinus Torvalds .init = route4_init, 6651da177e4SLinus Torvalds .destroy = route4_destroy, 6661da177e4SLinus Torvalds .get = route4_get, 6671da177e4SLinus Torvalds .change = route4_change, 6681da177e4SLinus Torvalds .delete = route4_delete, 6691da177e4SLinus Torvalds .walk = route4_walk, 6701da177e4SLinus Torvalds .dump = route4_dump, 67107d79fc7SCong Wang .bind_class = route4_bind_class, 6721da177e4SLinus Torvalds .owner = THIS_MODULE, 6731da177e4SLinus Torvalds }; 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds static int __init init_route4(void) 6761da177e4SLinus Torvalds { 6771da177e4SLinus Torvalds return register_tcf_proto_ops(&cls_route4_ops); 6781da177e4SLinus Torvalds } 6791da177e4SLinus Torvalds 6801da177e4SLinus Torvalds static void __exit exit_route4(void) 6811da177e4SLinus Torvalds { 6821da177e4SLinus Torvalds unregister_tcf_proto_ops(&cls_route4_ops); 6831da177e4SLinus Torvalds } 6841da177e4SLinus Torvalds 6851da177e4SLinus Torvalds module_init(init_route4) 6861da177e4SLinus Torvalds module_exit(exit_route4) 6871da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 688