1b57dc7c1SPaul Blakey // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2b57dc7c1SPaul Blakey /* -
3b57dc7c1SPaul Blakey * net/sched/act_ct.c Connection Tracking action
4b57dc7c1SPaul Blakey *
5b57dc7c1SPaul Blakey * Authors: Paul Blakey <paulb@mellanox.com>
6b57dc7c1SPaul Blakey * Yossi Kuperman <yossiku@mellanox.com>
7b57dc7c1SPaul Blakey * Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
8b57dc7c1SPaul Blakey */
9b57dc7c1SPaul Blakey
10b57dc7c1SPaul Blakey #include <linux/module.h>
11b57dc7c1SPaul Blakey #include <linux/init.h>
12b57dc7c1SPaul Blakey #include <linux/kernel.h>
13b57dc7c1SPaul Blakey #include <linux/skbuff.h>
14b57dc7c1SPaul Blakey #include <linux/rtnetlink.h>
15b57dc7c1SPaul Blakey #include <linux/pkt_cls.h>
16b57dc7c1SPaul Blakey #include <linux/ip.h>
17b57dc7c1SPaul Blakey #include <linux/ipv6.h>
18c34b961aSPaul Blakey #include <linux/rhashtable.h>
19b57dc7c1SPaul Blakey #include <net/netlink.h>
20b57dc7c1SPaul Blakey #include <net/pkt_sched.h>
21b57dc7c1SPaul Blakey #include <net/pkt_cls.h>
22b57dc7c1SPaul Blakey #include <net/act_api.h>
23b57dc7c1SPaul Blakey #include <net/ip.h>
24b57dc7c1SPaul Blakey #include <net/ipv6_frag.h>
25b57dc7c1SPaul Blakey #include <uapi/linux/tc_act/tc_ct.h>
26b57dc7c1SPaul Blakey #include <net/tc_act/tc_ct.h>
27871cf386SPedro Tammela #include <net/tc_wrapper.h>
28b57dc7c1SPaul Blakey
29c34b961aSPaul Blakey #include <net/netfilter/nf_flow_table.h>
30b57dc7c1SPaul Blakey #include <net/netfilter/nf_conntrack.h>
31b57dc7c1SPaul Blakey #include <net/netfilter/nf_conntrack_core.h>
32b57dc7c1SPaul Blakey #include <net/netfilter/nf_conntrack_zones.h>
33b57dc7c1SPaul Blakey #include <net/netfilter/nf_conntrack_helper.h>
34beb97d3aSwenxu #include <net/netfilter/nf_conntrack_acct.h>
35b57dc7c1SPaul Blakey #include <net/netfilter/ipv6/nf_defrag_ipv6.h>
369795ded7SPaul Blakey #include <net/netfilter/nf_conntrack_act_ct.h>
37a21b06e7SXin Long #include <net/netfilter/nf_conntrack_seqadj.h>
3840d102cdSJeremy Sowden #include <uapi/linux/netfilter/nf_nat.h>
39b57dc7c1SPaul Blakey
40c34b961aSPaul Blakey static struct workqueue_struct *act_ct_wq;
41c34b961aSPaul Blakey static struct rhashtable zones_ht;
42138470a9SEric Dumazet static DEFINE_MUTEX(zones_mutex);
43c34b961aSPaul Blakey
44*9126fd82SXin Long struct zones_ht_key {
45*9126fd82SXin Long struct net *net;
46*9126fd82SXin Long u16 zone;
47*9126fd82SXin Long };
48*9126fd82SXin Long
49c34b961aSPaul Blakey struct tcf_ct_flow_table {
50c34b961aSPaul Blakey struct rhash_head node; /* In zones tables */
51c34b961aSPaul Blakey
52c34b961aSPaul Blakey struct rcu_work rwork;
53c34b961aSPaul Blakey struct nf_flowtable nf_ft;
54138470a9SEric Dumazet refcount_t ref;
55*9126fd82SXin Long struct zones_ht_key key;
56c34b961aSPaul Blakey
57c34b961aSPaul Blakey bool dying;
58c34b961aSPaul Blakey };
59c34b961aSPaul Blakey
60c34b961aSPaul Blakey static const struct rhashtable_params zones_params = {
61c34b961aSPaul Blakey .head_offset = offsetof(struct tcf_ct_flow_table, node),
62*9126fd82SXin Long .key_offset = offsetof(struct tcf_ct_flow_table, key),
63*9126fd82SXin Long .key_len = sizeof_field(struct tcf_ct_flow_table, key),
64c34b961aSPaul Blakey .automatic_shrinking = true,
65c34b961aSPaul Blakey };
66c34b961aSPaul Blakey
679c26ba9bSPaul Blakey static struct flow_action_entry *
tcf_ct_flow_table_flow_action_get_next(struct flow_action * flow_action)689c26ba9bSPaul Blakey tcf_ct_flow_table_flow_action_get_next(struct flow_action *flow_action)
699c26ba9bSPaul Blakey {
709c26ba9bSPaul Blakey int i = flow_action->num_entries++;
719c26ba9bSPaul Blakey
729c26ba9bSPaul Blakey return &flow_action->entries[i];
739c26ba9bSPaul Blakey }
749c26ba9bSPaul Blakey
tcf_ct_add_mangle_action(struct flow_action * action,enum flow_action_mangle_base htype,u32 offset,u32 mask,u32 val)759c26ba9bSPaul Blakey static void tcf_ct_add_mangle_action(struct flow_action *action,
769c26ba9bSPaul Blakey enum flow_action_mangle_base htype,
779c26ba9bSPaul Blakey u32 offset,
789c26ba9bSPaul Blakey u32 mask,
799c26ba9bSPaul Blakey u32 val)
809c26ba9bSPaul Blakey {
819c26ba9bSPaul Blakey struct flow_action_entry *entry;
829c26ba9bSPaul Blakey
839c26ba9bSPaul Blakey entry = tcf_ct_flow_table_flow_action_get_next(action);
849c26ba9bSPaul Blakey entry->id = FLOW_ACTION_MANGLE;
859c26ba9bSPaul Blakey entry->mangle.htype = htype;
869c26ba9bSPaul Blakey entry->mangle.mask = ~mask;
879c26ba9bSPaul Blakey entry->mangle.offset = offset;
889c26ba9bSPaul Blakey entry->mangle.val = val;
899c26ba9bSPaul Blakey }
909c26ba9bSPaul Blakey
919c26ba9bSPaul Blakey /* The following nat helper functions check if the inverted reverse tuple
929c26ba9bSPaul Blakey * (target) is different then the current dir tuple - meaning nat for ports
939c26ba9bSPaul Blakey * and/or ip is needed, and add the relevant mangle actions.
949c26ba9bSPaul Blakey */
959c26ba9bSPaul Blakey static void
tcf_ct_flow_table_add_action_nat_ipv4(const struct nf_conntrack_tuple * tuple,struct nf_conntrack_tuple target,struct flow_action * action)969c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_ipv4(const struct nf_conntrack_tuple *tuple,
979c26ba9bSPaul Blakey struct nf_conntrack_tuple target,
989c26ba9bSPaul Blakey struct flow_action *action)
999c26ba9bSPaul Blakey {
1009c26ba9bSPaul Blakey if (memcmp(&target.src.u3, &tuple->src.u3, sizeof(target.src.u3)))
1019c26ba9bSPaul Blakey tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP4,
1029c26ba9bSPaul Blakey offsetof(struct iphdr, saddr),
1039c26ba9bSPaul Blakey 0xFFFFFFFF,
1049c26ba9bSPaul Blakey be32_to_cpu(target.src.u3.ip));
1059c26ba9bSPaul Blakey if (memcmp(&target.dst.u3, &tuple->dst.u3, sizeof(target.dst.u3)))
1069c26ba9bSPaul Blakey tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP4,
1079c26ba9bSPaul Blakey offsetof(struct iphdr, daddr),
1089c26ba9bSPaul Blakey 0xFFFFFFFF,
1099c26ba9bSPaul Blakey be32_to_cpu(target.dst.u3.ip));
1109c26ba9bSPaul Blakey }
1119c26ba9bSPaul Blakey
1129c26ba9bSPaul Blakey static void
tcf_ct_add_ipv6_addr_mangle_action(struct flow_action * action,union nf_inet_addr * addr,u32 offset)1139c26ba9bSPaul Blakey tcf_ct_add_ipv6_addr_mangle_action(struct flow_action *action,
1149c26ba9bSPaul Blakey union nf_inet_addr *addr,
1159c26ba9bSPaul Blakey u32 offset)
1169c26ba9bSPaul Blakey {
1179c26ba9bSPaul Blakey int i;
1189c26ba9bSPaul Blakey
1199c26ba9bSPaul Blakey for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++)
1209c26ba9bSPaul Blakey tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
1219c26ba9bSPaul Blakey i * sizeof(u32) + offset,
1229c26ba9bSPaul Blakey 0xFFFFFFFF, be32_to_cpu(addr->ip6[i]));
1239c26ba9bSPaul Blakey }
1249c26ba9bSPaul Blakey
1259c26ba9bSPaul Blakey static void
tcf_ct_flow_table_add_action_nat_ipv6(const struct nf_conntrack_tuple * tuple,struct nf_conntrack_tuple target,struct flow_action * action)1269c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_ipv6(const struct nf_conntrack_tuple *tuple,
1279c26ba9bSPaul Blakey struct nf_conntrack_tuple target,
1289c26ba9bSPaul Blakey struct flow_action *action)
1299c26ba9bSPaul Blakey {
1309c26ba9bSPaul Blakey if (memcmp(&target.src.u3, &tuple->src.u3, sizeof(target.src.u3)))
1319c26ba9bSPaul Blakey tcf_ct_add_ipv6_addr_mangle_action(action, &target.src.u3,
1329c26ba9bSPaul Blakey offsetof(struct ipv6hdr,
1339c26ba9bSPaul Blakey saddr));
1349c26ba9bSPaul Blakey if (memcmp(&target.dst.u3, &tuple->dst.u3, sizeof(target.dst.u3)))
1359c26ba9bSPaul Blakey tcf_ct_add_ipv6_addr_mangle_action(action, &target.dst.u3,
1369c26ba9bSPaul Blakey offsetof(struct ipv6hdr,
1379c26ba9bSPaul Blakey daddr));
1389c26ba9bSPaul Blakey }
1399c26ba9bSPaul Blakey
1409c26ba9bSPaul Blakey static void
tcf_ct_flow_table_add_action_nat_tcp(const struct nf_conntrack_tuple * tuple,struct nf_conntrack_tuple target,struct flow_action * action)1419c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_tcp(const struct nf_conntrack_tuple *tuple,
1429c26ba9bSPaul Blakey struct nf_conntrack_tuple target,
1439c26ba9bSPaul Blakey struct flow_action *action)
1449c26ba9bSPaul Blakey {
1459c26ba9bSPaul Blakey __be16 target_src = target.src.u.tcp.port;
1469c26ba9bSPaul Blakey __be16 target_dst = target.dst.u.tcp.port;
1479c26ba9bSPaul Blakey
1489c26ba9bSPaul Blakey if (target_src != tuple->src.u.tcp.port)
1499c26ba9bSPaul Blakey tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_TCP,
1509c26ba9bSPaul Blakey offsetof(struct tcphdr, source),
1519c26ba9bSPaul Blakey 0xFFFF, be16_to_cpu(target_src));
1529c26ba9bSPaul Blakey if (target_dst != tuple->dst.u.tcp.port)
1539c26ba9bSPaul Blakey tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_TCP,
1549c26ba9bSPaul Blakey offsetof(struct tcphdr, dest),
1559c26ba9bSPaul Blakey 0xFFFF, be16_to_cpu(target_dst));
1569c26ba9bSPaul Blakey }
1579c26ba9bSPaul Blakey
1589c26ba9bSPaul Blakey static void
tcf_ct_flow_table_add_action_nat_udp(const struct nf_conntrack_tuple * tuple,struct nf_conntrack_tuple target,struct flow_action * action)1599c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_udp(const struct nf_conntrack_tuple *tuple,
1609c26ba9bSPaul Blakey struct nf_conntrack_tuple target,
1619c26ba9bSPaul Blakey struct flow_action *action)
1629c26ba9bSPaul Blakey {
1639c26ba9bSPaul Blakey __be16 target_src = target.src.u.udp.port;
1649c26ba9bSPaul Blakey __be16 target_dst = target.dst.u.udp.port;
1659c26ba9bSPaul Blakey
1669c26ba9bSPaul Blakey if (target_src != tuple->src.u.udp.port)
16747b5d2a1SRoi Dayan tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_UDP,
1689c26ba9bSPaul Blakey offsetof(struct udphdr, source),
1699c26ba9bSPaul Blakey 0xFFFF, be16_to_cpu(target_src));
1709c26ba9bSPaul Blakey if (target_dst != tuple->dst.u.udp.port)
17147b5d2a1SRoi Dayan tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_UDP,
1729c26ba9bSPaul Blakey offsetof(struct udphdr, dest),
1739c26ba9bSPaul Blakey 0xFFFF, be16_to_cpu(target_dst));
1749c26ba9bSPaul Blakey }
1759c26ba9bSPaul Blakey
tcf_ct_flow_table_add_action_meta(struct nf_conn * ct,enum ip_conntrack_dir dir,enum ip_conntrack_info ctinfo,struct flow_action * action)1769c26ba9bSPaul Blakey static void tcf_ct_flow_table_add_action_meta(struct nf_conn *ct,
1779c26ba9bSPaul Blakey enum ip_conntrack_dir dir,
1781a441a9bSVlad Buslov enum ip_conntrack_info ctinfo,
1799c26ba9bSPaul Blakey struct flow_action *action)
1809c26ba9bSPaul Blakey {
1819c26ba9bSPaul Blakey struct nf_conn_labels *ct_labels;
1829c26ba9bSPaul Blakey struct flow_action_entry *entry;
1839c26ba9bSPaul Blakey u32 *act_ct_labels;
1849c26ba9bSPaul Blakey
1859c26ba9bSPaul Blakey entry = tcf_ct_flow_table_flow_action_get_next(action);
1869c26ba9bSPaul Blakey entry->id = FLOW_ACTION_CT_METADATA;
1879c26ba9bSPaul Blakey #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
18852d1aa8bSDaniel Xu entry->ct_metadata.mark = READ_ONCE(ct->mark);
1899c26ba9bSPaul Blakey #endif
19030b0cf90SPaul Blakey /* aligns with the CT reference on the SKB nf_ct_set */
19130b0cf90SPaul Blakey entry->ct_metadata.cookie = (unsigned long)ct | ctinfo;
192941eff5aSPaul Blakey entry->ct_metadata.orig_dir = dir == IP_CT_DIR_ORIGINAL;
1939c26ba9bSPaul Blakey
1949c26ba9bSPaul Blakey act_ct_labels = entry->ct_metadata.labels;
1959c26ba9bSPaul Blakey ct_labels = nf_ct_labels_find(ct);
1969c26ba9bSPaul Blakey if (ct_labels)
1979c26ba9bSPaul Blakey memcpy(act_ct_labels, ct_labels->bits, NF_CT_LABELS_MAX_SIZE);
1989c26ba9bSPaul Blakey else
1999c26ba9bSPaul Blakey memset(act_ct_labels, 0, NF_CT_LABELS_MAX_SIZE);
2009c26ba9bSPaul Blakey }
2019c26ba9bSPaul Blakey
tcf_ct_flow_table_add_action_nat(struct net * net,struct nf_conn * ct,enum ip_conntrack_dir dir,struct flow_action * action)2029c26ba9bSPaul Blakey static int tcf_ct_flow_table_add_action_nat(struct net *net,
2039c26ba9bSPaul Blakey struct nf_conn *ct,
2049c26ba9bSPaul Blakey enum ip_conntrack_dir dir,
2059c26ba9bSPaul Blakey struct flow_action *action)
2069c26ba9bSPaul Blakey {
2079c26ba9bSPaul Blakey const struct nf_conntrack_tuple *tuple = &ct->tuplehash[dir].tuple;
2089c26ba9bSPaul Blakey struct nf_conntrack_tuple target;
2099c26ba9bSPaul Blakey
21005aa69e5Swenxu if (!(ct->status & IPS_NAT_MASK))
21105aa69e5Swenxu return 0;
21205aa69e5Swenxu
2139c26ba9bSPaul Blakey nf_ct_invert_tuple(&target, &ct->tuplehash[!dir].tuple);
2149c26ba9bSPaul Blakey
2159c26ba9bSPaul Blakey switch (tuple->src.l3num) {
2169c26ba9bSPaul Blakey case NFPROTO_IPV4:
2179c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_ipv4(tuple, target,
2189c26ba9bSPaul Blakey action);
2199c26ba9bSPaul Blakey break;
2209c26ba9bSPaul Blakey case NFPROTO_IPV6:
2219c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_ipv6(tuple, target,
2229c26ba9bSPaul Blakey action);
2239c26ba9bSPaul Blakey break;
2249c26ba9bSPaul Blakey default:
2259c26ba9bSPaul Blakey return -EOPNOTSUPP;
2269c26ba9bSPaul Blakey }
2279c26ba9bSPaul Blakey
2289c26ba9bSPaul Blakey switch (nf_ct_protonum(ct)) {
2299c26ba9bSPaul Blakey case IPPROTO_TCP:
2309c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_tcp(tuple, target, action);
2319c26ba9bSPaul Blakey break;
2329c26ba9bSPaul Blakey case IPPROTO_UDP:
2339c26ba9bSPaul Blakey tcf_ct_flow_table_add_action_nat_udp(tuple, target, action);
2349c26ba9bSPaul Blakey break;
2359c26ba9bSPaul Blakey default:
2369c26ba9bSPaul Blakey return -EOPNOTSUPP;
2379c26ba9bSPaul Blakey }
2389c26ba9bSPaul Blakey
2399c26ba9bSPaul Blakey return 0;
2409c26ba9bSPaul Blakey }
2419c26ba9bSPaul Blakey
tcf_ct_flow_table_fill_actions(struct net * net,struct flow_offload * flow,enum flow_offload_tuple_dir tdir,struct nf_flow_rule * flow_rule)2429c26ba9bSPaul Blakey static int tcf_ct_flow_table_fill_actions(struct net *net,
2431a441a9bSVlad Buslov struct flow_offload *flow,
2449c26ba9bSPaul Blakey enum flow_offload_tuple_dir tdir,
2459c26ba9bSPaul Blakey struct nf_flow_rule *flow_rule)
2469c26ba9bSPaul Blakey {
2479c26ba9bSPaul Blakey struct flow_action *action = &flow_rule->rule->action;
2489c26ba9bSPaul Blakey int num_entries = action->num_entries;
2499c26ba9bSPaul Blakey struct nf_conn *ct = flow->ct;
2501a441a9bSVlad Buslov enum ip_conntrack_info ctinfo;
2519c26ba9bSPaul Blakey enum ip_conntrack_dir dir;
2529c26ba9bSPaul Blakey int i, err;
2539c26ba9bSPaul Blakey
2549c26ba9bSPaul Blakey switch (tdir) {
2559c26ba9bSPaul Blakey case FLOW_OFFLOAD_DIR_ORIGINAL:
2569c26ba9bSPaul Blakey dir = IP_CT_DIR_ORIGINAL;
257d5774cb6SVlad Buslov ctinfo = test_bit(IPS_SEEN_REPLY_BIT, &ct->status) ?
258d5774cb6SVlad Buslov IP_CT_ESTABLISHED : IP_CT_NEW;
259d5774cb6SVlad Buslov if (ctinfo == IP_CT_ESTABLISHED)
2601a441a9bSVlad Buslov set_bit(NF_FLOW_HW_ESTABLISHED, &flow->flags);
2619c26ba9bSPaul Blakey break;
2629c26ba9bSPaul Blakey case FLOW_OFFLOAD_DIR_REPLY:
2639c26ba9bSPaul Blakey dir = IP_CT_DIR_REPLY;
2641a441a9bSVlad Buslov ctinfo = IP_CT_ESTABLISHED_REPLY;
2659c26ba9bSPaul Blakey break;
2669c26ba9bSPaul Blakey default:
2679c26ba9bSPaul Blakey return -EOPNOTSUPP;
2689c26ba9bSPaul Blakey }
2699c26ba9bSPaul Blakey
2709c26ba9bSPaul Blakey err = tcf_ct_flow_table_add_action_nat(net, ct, dir, action);
2719c26ba9bSPaul Blakey if (err)
2729c26ba9bSPaul Blakey goto err_nat;
2739c26ba9bSPaul Blakey
2741a441a9bSVlad Buslov tcf_ct_flow_table_add_action_meta(ct, dir, ctinfo, action);
2759c26ba9bSPaul Blakey return 0;
2769c26ba9bSPaul Blakey
2779c26ba9bSPaul Blakey err_nat:
2789c26ba9bSPaul Blakey /* Clear filled actions */
2799c26ba9bSPaul Blakey for (i = num_entries; i < action->num_entries; i++)
2809c26ba9bSPaul Blakey memset(&action->entries[i], 0, sizeof(action->entries[i]));
2819c26ba9bSPaul Blakey action->num_entries = num_entries;
2829c26ba9bSPaul Blakey
2839c26ba9bSPaul Blakey return err;
2849c26ba9bSPaul Blakey }
2859c26ba9bSPaul Blakey
tcf_ct_flow_is_outdated(const struct flow_offload * flow)286735795f6SPablo Neira Ayuso static bool tcf_ct_flow_is_outdated(const struct flow_offload *flow)
287735795f6SPablo Neira Ayuso {
288735795f6SPablo Neira Ayuso return test_bit(IPS_SEEN_REPLY_BIT, &flow->ct->status) &&
289a63b6622SVlad Buslov test_bit(IPS_HW_OFFLOAD_BIT, &flow->ct->status) &&
290a63b6622SVlad Buslov !test_bit(NF_FLOW_HW_PENDING, &flow->flags) &&
291735795f6SPablo Neira Ayuso !test_bit(NF_FLOW_HW_ESTABLISHED, &flow->flags);
292735795f6SPablo Neira Ayuso }
293735795f6SPablo Neira Ayuso
29415f300edSVlad Buslov static void tcf_ct_flow_table_get_ref(struct tcf_ct_flow_table *ct_ft);
29515f300edSVlad Buslov
tcf_ct_nf_get(struct nf_flowtable * ft)29615f300edSVlad Buslov static void tcf_ct_nf_get(struct nf_flowtable *ft)
29715f300edSVlad Buslov {
29815f300edSVlad Buslov struct tcf_ct_flow_table *ct_ft =
29915f300edSVlad Buslov container_of(ft, struct tcf_ct_flow_table, nf_ft);
30015f300edSVlad Buslov
30115f300edSVlad Buslov tcf_ct_flow_table_get_ref(ct_ft);
30215f300edSVlad Buslov }
30315f300edSVlad Buslov
30415f300edSVlad Buslov static void tcf_ct_flow_table_put(struct tcf_ct_flow_table *ct_ft);
30515f300edSVlad Buslov
tcf_ct_nf_put(struct nf_flowtable * ft)30615f300edSVlad Buslov static void tcf_ct_nf_put(struct nf_flowtable *ft)
30715f300edSVlad Buslov {
30815f300edSVlad Buslov struct tcf_ct_flow_table *ct_ft =
30915f300edSVlad Buslov container_of(ft, struct tcf_ct_flow_table, nf_ft);
31015f300edSVlad Buslov
31115f300edSVlad Buslov tcf_ct_flow_table_put(ct_ft);
31215f300edSVlad Buslov }
31315f300edSVlad Buslov
314c34b961aSPaul Blakey static struct nf_flowtable_type flowtable_ct = {
315735795f6SPablo Neira Ayuso .gc = tcf_ct_flow_is_outdated,
3169c26ba9bSPaul Blakey .action = tcf_ct_flow_table_fill_actions,
31715f300edSVlad Buslov .get = tcf_ct_nf_get,
31815f300edSVlad Buslov .put = tcf_ct_nf_put,
319c34b961aSPaul Blakey .owner = THIS_MODULE,
320c34b961aSPaul Blakey };
321c34b961aSPaul Blakey
tcf_ct_flow_table_get(struct net * net,struct tcf_ct_params * params)322fc54d906SVlad Buslov static int tcf_ct_flow_table_get(struct net *net, struct tcf_ct_params *params)
323c34b961aSPaul Blakey {
324*9126fd82SXin Long struct zones_ht_key key = { .net = net, .zone = params->zone };
325c34b961aSPaul Blakey struct tcf_ct_flow_table *ct_ft;
326c34b961aSPaul Blakey int err = -ENOMEM;
327c34b961aSPaul Blakey
328138470a9SEric Dumazet mutex_lock(&zones_mutex);
329*9126fd82SXin Long ct_ft = rhashtable_lookup_fast(&zones_ht, &key, zones_params);
330138470a9SEric Dumazet if (ct_ft && refcount_inc_not_zero(&ct_ft->ref))
331138470a9SEric Dumazet goto out_unlock;
332c34b961aSPaul Blakey
333138470a9SEric Dumazet ct_ft = kzalloc(sizeof(*ct_ft), GFP_KERNEL);
334c34b961aSPaul Blakey if (!ct_ft)
335c34b961aSPaul Blakey goto err_alloc;
336138470a9SEric Dumazet refcount_set(&ct_ft->ref, 1);
337c34b961aSPaul Blakey
338*9126fd82SXin Long ct_ft->key = key;
339c34b961aSPaul Blakey err = rhashtable_insert_fast(&zones_ht, &ct_ft->node, zones_params);
340c34b961aSPaul Blakey if (err)
341c34b961aSPaul Blakey goto err_insert;
342c34b961aSPaul Blakey
343c34b961aSPaul Blakey ct_ft->nf_ft.type = &flowtable_ct;
3443567e233SMarcelo Ricardo Leitner ct_ft->nf_ft.flags |= NF_FLOWTABLE_HW_OFFLOAD |
3453567e233SMarcelo Ricardo Leitner NF_FLOWTABLE_COUNTER;
346c34b961aSPaul Blakey err = nf_flow_table_init(&ct_ft->nf_ft);
347c34b961aSPaul Blakey if (err)
348c34b961aSPaul Blakey goto err_init;
349fc54d906SVlad Buslov write_pnet(&ct_ft->nf_ft.net, net);
350c34b961aSPaul Blakey
351c34b961aSPaul Blakey __module_get(THIS_MODULE);
352138470a9SEric Dumazet out_unlock:
353c34b961aSPaul Blakey params->ct_ft = ct_ft;
354edd5861eSPaul Blakey params->nf_ft = &ct_ft->nf_ft;
355138470a9SEric Dumazet mutex_unlock(&zones_mutex);
356c34b961aSPaul Blakey
357c34b961aSPaul Blakey return 0;
358c34b961aSPaul Blakey
359c34b961aSPaul Blakey err_init:
360c34b961aSPaul Blakey rhashtable_remove_fast(&zones_ht, &ct_ft->node, zones_params);
361c34b961aSPaul Blakey err_insert:
362c34b961aSPaul Blakey kfree(ct_ft);
363c34b961aSPaul Blakey err_alloc:
364138470a9SEric Dumazet mutex_unlock(&zones_mutex);
365c34b961aSPaul Blakey return err;
366c34b961aSPaul Blakey }
367c34b961aSPaul Blakey
tcf_ct_flow_table_get_ref(struct tcf_ct_flow_table * ct_ft)36815f300edSVlad Buslov static void tcf_ct_flow_table_get_ref(struct tcf_ct_flow_table *ct_ft)
36915f300edSVlad Buslov {
37015f300edSVlad Buslov refcount_inc(&ct_ft->ref);
37115f300edSVlad Buslov }
37215f300edSVlad Buslov
tcf_ct_flow_table_cleanup_work(struct work_struct * work)373c34b961aSPaul Blakey static void tcf_ct_flow_table_cleanup_work(struct work_struct *work)
374c34b961aSPaul Blakey {
375c34b961aSPaul Blakey struct tcf_ct_flow_table *ct_ft;
37677ac5e40SLouis Peens struct flow_block *block;
377c34b961aSPaul Blakey
378c34b961aSPaul Blakey ct_ft = container_of(to_rcu_work(work), struct tcf_ct_flow_table,
379c34b961aSPaul Blakey rwork);
380c34b961aSPaul Blakey nf_flow_table_free(&ct_ft->nf_ft);
38177ac5e40SLouis Peens
38277ac5e40SLouis Peens block = &ct_ft->nf_ft.flow_block;
38377ac5e40SLouis Peens down_write(&ct_ft->nf_ft.flow_block_lock);
38415f300edSVlad Buslov WARN_ON(!list_empty(&block->cb_list));
38577ac5e40SLouis Peens up_write(&ct_ft->nf_ft.flow_block_lock);
386c34b961aSPaul Blakey kfree(ct_ft);
387c34b961aSPaul Blakey
388c34b961aSPaul Blakey module_put(THIS_MODULE);
389c34b961aSPaul Blakey }
390c34b961aSPaul Blakey
tcf_ct_flow_table_put(struct tcf_ct_flow_table * ct_ft)39119138941SXin Long static void tcf_ct_flow_table_put(struct tcf_ct_flow_table *ct_ft)
392c34b961aSPaul Blakey {
39319138941SXin Long if (refcount_dec_and_test(&ct_ft->ref)) {
394c34b961aSPaul Blakey rhashtable_remove_fast(&zones_ht, &ct_ft->node, zones_params);
395c34b961aSPaul Blakey INIT_RCU_WORK(&ct_ft->rwork, tcf_ct_flow_table_cleanup_work);
396c34b961aSPaul Blakey queue_rcu_work(act_ct_wq, &ct_ft->rwork);
397c34b961aSPaul Blakey }
398c34b961aSPaul Blakey }
399c34b961aSPaul Blakey
tcf_ct_flow_tc_ifidx(struct flow_offload * entry,struct nf_conn_act_ct_ext * act_ct_ext,u8 dir)400db6140e5SPaul Blakey static void tcf_ct_flow_tc_ifidx(struct flow_offload *entry,
401db6140e5SPaul Blakey struct nf_conn_act_ct_ext *act_ct_ext, u8 dir)
402db6140e5SPaul Blakey {
403db6140e5SPaul Blakey entry->tuplehash[dir].tuple.xmit_type = FLOW_OFFLOAD_XMIT_TC;
404db6140e5SPaul Blakey entry->tuplehash[dir].tuple.tc.iifidx = act_ct_ext->ifindex[dir];
405db6140e5SPaul Blakey }
406db6140e5SPaul Blakey
tcf_ct_flow_ct_ext_ifidx_update(struct flow_offload * entry)407d5a116dbSVlad Buslov static void tcf_ct_flow_ct_ext_ifidx_update(struct flow_offload *entry)
408d5a116dbSVlad Buslov {
409d5a116dbSVlad Buslov struct nf_conn_act_ct_ext *act_ct_ext;
410d5a116dbSVlad Buslov
411d5a116dbSVlad Buslov act_ct_ext = nf_conn_act_ct_ext_find(entry->ct);
412d5a116dbSVlad Buslov if (act_ct_ext) {
413d5a116dbSVlad Buslov tcf_ct_flow_tc_ifidx(entry, act_ct_ext, FLOW_OFFLOAD_DIR_ORIGINAL);
414d5a116dbSVlad Buslov tcf_ct_flow_tc_ifidx(entry, act_ct_ext, FLOW_OFFLOAD_DIR_REPLY);
415d5a116dbSVlad Buslov }
416d5a116dbSVlad Buslov }
417d5a116dbSVlad Buslov
tcf_ct_flow_table_add(struct tcf_ct_flow_table * ct_ft,struct nf_conn * ct,bool tcp,bool bidirectional)41864ff70b8SPaul Blakey static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft,
41964ff70b8SPaul Blakey struct nf_conn *ct,
4206a9bad00SVlad Buslov bool tcp, bool bidirectional)
42164ff70b8SPaul Blakey {
4229795ded7SPaul Blakey struct nf_conn_act_ct_ext *act_ct_ext;
42364ff70b8SPaul Blakey struct flow_offload *entry;
42464ff70b8SPaul Blakey int err;
42564ff70b8SPaul Blakey
42664ff70b8SPaul Blakey if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
42764ff70b8SPaul Blakey return;
42864ff70b8SPaul Blakey
42964ff70b8SPaul Blakey entry = flow_offload_alloc(ct);
43064ff70b8SPaul Blakey if (!entry) {
43164ff70b8SPaul Blakey WARN_ON_ONCE(1);
43264ff70b8SPaul Blakey goto err_alloc;
43364ff70b8SPaul Blakey }
43464ff70b8SPaul Blakey
43564ff70b8SPaul Blakey if (tcp) {
43664ff70b8SPaul Blakey ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
43764ff70b8SPaul Blakey ct->proto.tcp.seen[1].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
43864ff70b8SPaul Blakey }
4396a9bad00SVlad Buslov if (bidirectional)
4406a9bad00SVlad Buslov __set_bit(NF_FLOW_HW_BIDIRECTIONAL, &entry->flags);
44164ff70b8SPaul Blakey
4429795ded7SPaul Blakey act_ct_ext = nf_conn_act_ct_ext_find(ct);
4439795ded7SPaul Blakey if (act_ct_ext) {
444db6140e5SPaul Blakey tcf_ct_flow_tc_ifidx(entry, act_ct_ext, FLOW_OFFLOAD_DIR_ORIGINAL);
445db6140e5SPaul Blakey tcf_ct_flow_tc_ifidx(entry, act_ct_ext, FLOW_OFFLOAD_DIR_REPLY);
4469795ded7SPaul Blakey }
4479795ded7SPaul Blakey
44864ff70b8SPaul Blakey err = flow_offload_add(&ct_ft->nf_ft, entry);
44964ff70b8SPaul Blakey if (err)
45064ff70b8SPaul Blakey goto err_add;
45164ff70b8SPaul Blakey
45264ff70b8SPaul Blakey return;
45364ff70b8SPaul Blakey
45464ff70b8SPaul Blakey err_add:
45564ff70b8SPaul Blakey flow_offload_free(entry);
45664ff70b8SPaul Blakey err_alloc:
45764ff70b8SPaul Blakey clear_bit(IPS_OFFLOAD_BIT, &ct->status);
45864ff70b8SPaul Blakey }
45964ff70b8SPaul Blakey
tcf_ct_flow_table_process_conn(struct tcf_ct_flow_table * ct_ft,struct nf_conn * ct,enum ip_conntrack_info ctinfo)46064ff70b8SPaul Blakey static void tcf_ct_flow_table_process_conn(struct tcf_ct_flow_table *ct_ft,
46164ff70b8SPaul Blakey struct nf_conn *ct,
46264ff70b8SPaul Blakey enum ip_conntrack_info ctinfo)
46364ff70b8SPaul Blakey {
4646a9bad00SVlad Buslov bool tcp = false, bidirectional = true;
46564ff70b8SPaul Blakey
46664ff70b8SPaul Blakey switch (nf_ct_protonum(ct)) {
46764ff70b8SPaul Blakey case IPPROTO_TCP:
4686a9bad00SVlad Buslov if ((ctinfo != IP_CT_ESTABLISHED &&
4696a9bad00SVlad Buslov ctinfo != IP_CT_ESTABLISHED_REPLY) ||
4706a9bad00SVlad Buslov !test_bit(IPS_ASSURED_BIT, &ct->status) ||
4716a9bad00SVlad Buslov ct->proto.tcp.state != TCP_CONNTRACK_ESTABLISHED)
47264ff70b8SPaul Blakey return;
4736a9bad00SVlad Buslov
4746a9bad00SVlad Buslov tcp = true;
47564ff70b8SPaul Blakey break;
47664ff70b8SPaul Blakey case IPPROTO_UDP:
4776a9bad00SVlad Buslov if (!nf_ct_is_confirmed(ct))
4786a9bad00SVlad Buslov return;
4796a9bad00SVlad Buslov if (!test_bit(IPS_ASSURED_BIT, &ct->status))
4806a9bad00SVlad Buslov bidirectional = false;
48164ff70b8SPaul Blakey break;
482fcb6aa86SToshiaki Makita #ifdef CONFIG_NF_CT_PROTO_GRE
483fcb6aa86SToshiaki Makita case IPPROTO_GRE: {
484fcb6aa86SToshiaki Makita struct nf_conntrack_tuple *tuple;
485fcb6aa86SToshiaki Makita
4866a9bad00SVlad Buslov if ((ctinfo != IP_CT_ESTABLISHED &&
4876a9bad00SVlad Buslov ctinfo != IP_CT_ESTABLISHED_REPLY) ||
4886a9bad00SVlad Buslov !test_bit(IPS_ASSURED_BIT, &ct->status) ||
4896a9bad00SVlad Buslov ct->status & IPS_NAT_MASK)
490fcb6aa86SToshiaki Makita return;
4916a9bad00SVlad Buslov
492fcb6aa86SToshiaki Makita tuple = &ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple;
493fcb6aa86SToshiaki Makita /* No support for GRE v1 */
494fcb6aa86SToshiaki Makita if (tuple->src.u.gre.key || tuple->dst.u.gre.key)
495fcb6aa86SToshiaki Makita return;
496fcb6aa86SToshiaki Makita break;
497fcb6aa86SToshiaki Makita }
498fcb6aa86SToshiaki Makita #endif
49964ff70b8SPaul Blakey default:
50064ff70b8SPaul Blakey return;
50164ff70b8SPaul Blakey }
50264ff70b8SPaul Blakey
50364ff70b8SPaul Blakey if (nf_ct_ext_exist(ct, NF_CT_EXT_HELPER) ||
50464ff70b8SPaul Blakey ct->status & IPS_SEQ_ADJUST)
50564ff70b8SPaul Blakey return;
50664ff70b8SPaul Blakey
5076a9bad00SVlad Buslov tcf_ct_flow_table_add(ct_ft, ct, tcp, bidirectional);
50864ff70b8SPaul Blakey }
50964ff70b8SPaul Blakey
51046475bb2SPaul Blakey static bool
tcf_ct_flow_table_fill_tuple_ipv4(struct sk_buff * skb,struct flow_offload_tuple * tuple,struct tcphdr ** tcph)51146475bb2SPaul Blakey tcf_ct_flow_table_fill_tuple_ipv4(struct sk_buff *skb,
51207ac9d16SPaul Blakey struct flow_offload_tuple *tuple,
51307ac9d16SPaul Blakey struct tcphdr **tcph)
51446475bb2SPaul Blakey {
51546475bb2SPaul Blakey struct flow_ports *ports;
51646475bb2SPaul Blakey unsigned int thoff;
51746475bb2SPaul Blakey struct iphdr *iph;
518fcb6aa86SToshiaki Makita size_t hdrsize;
519fcb6aa86SToshiaki Makita u8 ipproto;
52046475bb2SPaul Blakey
5214cc5fdecSPaul Blakey if (!pskb_network_may_pull(skb, sizeof(*iph)))
52246475bb2SPaul Blakey return false;
52346475bb2SPaul Blakey
52446475bb2SPaul Blakey iph = ip_hdr(skb);
52546475bb2SPaul Blakey thoff = iph->ihl * 4;
52646475bb2SPaul Blakey
52746475bb2SPaul Blakey if (ip_is_fragment(iph) ||
52846475bb2SPaul Blakey unlikely(thoff != sizeof(struct iphdr)))
52946475bb2SPaul Blakey return false;
53046475bb2SPaul Blakey
531fcb6aa86SToshiaki Makita ipproto = iph->protocol;
532fcb6aa86SToshiaki Makita switch (ipproto) {
533fcb6aa86SToshiaki Makita case IPPROTO_TCP:
534fcb6aa86SToshiaki Makita hdrsize = sizeof(struct tcphdr);
535fcb6aa86SToshiaki Makita break;
536fcb6aa86SToshiaki Makita case IPPROTO_UDP:
537fcb6aa86SToshiaki Makita hdrsize = sizeof(*ports);
538fcb6aa86SToshiaki Makita break;
539fcb6aa86SToshiaki Makita #ifdef CONFIG_NF_CT_PROTO_GRE
540fcb6aa86SToshiaki Makita case IPPROTO_GRE:
541fcb6aa86SToshiaki Makita hdrsize = sizeof(struct gre_base_hdr);
542fcb6aa86SToshiaki Makita break;
543fcb6aa86SToshiaki Makita #endif
544fcb6aa86SToshiaki Makita default:
54546475bb2SPaul Blakey return false;
546fcb6aa86SToshiaki Makita }
54746475bb2SPaul Blakey
54846475bb2SPaul Blakey if (iph->ttl <= 1)
54946475bb2SPaul Blakey return false;
55046475bb2SPaul Blakey
551fcb6aa86SToshiaki Makita if (!pskb_network_may_pull(skb, thoff + hdrsize))
55246475bb2SPaul Blakey return false;
55346475bb2SPaul Blakey
554fcb6aa86SToshiaki Makita switch (ipproto) {
555fcb6aa86SToshiaki Makita case IPPROTO_TCP:
55607ac9d16SPaul Blakey *tcph = (void *)(skb_network_header(skb) + thoff);
557fcb6aa86SToshiaki Makita fallthrough;
558fcb6aa86SToshiaki Makita case IPPROTO_UDP:
55907ac9d16SPaul Blakey ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
56046475bb2SPaul Blakey tuple->src_port = ports->source;
56146475bb2SPaul Blakey tuple->dst_port = ports->dest;
562fcb6aa86SToshiaki Makita break;
563fcb6aa86SToshiaki Makita case IPPROTO_GRE: {
564fcb6aa86SToshiaki Makita struct gre_base_hdr *greh;
565fcb6aa86SToshiaki Makita
566fcb6aa86SToshiaki Makita greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
567fcb6aa86SToshiaki Makita if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
568fcb6aa86SToshiaki Makita return false;
569fcb6aa86SToshiaki Makita break;
570fcb6aa86SToshiaki Makita }
571fcb6aa86SToshiaki Makita }
572fcb6aa86SToshiaki Makita
573fcb6aa86SToshiaki Makita iph = ip_hdr(skb);
574fcb6aa86SToshiaki Makita
575fcb6aa86SToshiaki Makita tuple->src_v4.s_addr = iph->saddr;
576fcb6aa86SToshiaki Makita tuple->dst_v4.s_addr = iph->daddr;
57746475bb2SPaul Blakey tuple->l3proto = AF_INET;
578fcb6aa86SToshiaki Makita tuple->l4proto = ipproto;
57946475bb2SPaul Blakey
58046475bb2SPaul Blakey return true;
58146475bb2SPaul Blakey }
58246475bb2SPaul Blakey
58346475bb2SPaul Blakey static bool
tcf_ct_flow_table_fill_tuple_ipv6(struct sk_buff * skb,struct flow_offload_tuple * tuple,struct tcphdr ** tcph)58446475bb2SPaul Blakey tcf_ct_flow_table_fill_tuple_ipv6(struct sk_buff *skb,
58507ac9d16SPaul Blakey struct flow_offload_tuple *tuple,
58607ac9d16SPaul Blakey struct tcphdr **tcph)
58746475bb2SPaul Blakey {
58846475bb2SPaul Blakey struct flow_ports *ports;
58946475bb2SPaul Blakey struct ipv6hdr *ip6h;
59046475bb2SPaul Blakey unsigned int thoff;
591fcb6aa86SToshiaki Makita size_t hdrsize;
592fcb6aa86SToshiaki Makita u8 nexthdr;
59346475bb2SPaul Blakey
5944cc5fdecSPaul Blakey if (!pskb_network_may_pull(skb, sizeof(*ip6h)))
59546475bb2SPaul Blakey return false;
59646475bb2SPaul Blakey
59746475bb2SPaul Blakey ip6h = ipv6_hdr(skb);
598fcb6aa86SToshiaki Makita thoff = sizeof(*ip6h);
59946475bb2SPaul Blakey
600fcb6aa86SToshiaki Makita nexthdr = ip6h->nexthdr;
601fcb6aa86SToshiaki Makita switch (nexthdr) {
602fcb6aa86SToshiaki Makita case IPPROTO_TCP:
603fcb6aa86SToshiaki Makita hdrsize = sizeof(struct tcphdr);
604fcb6aa86SToshiaki Makita break;
605fcb6aa86SToshiaki Makita case IPPROTO_UDP:
606fcb6aa86SToshiaki Makita hdrsize = sizeof(*ports);
607fcb6aa86SToshiaki Makita break;
608fcb6aa86SToshiaki Makita #ifdef CONFIG_NF_CT_PROTO_GRE
609fcb6aa86SToshiaki Makita case IPPROTO_GRE:
610fcb6aa86SToshiaki Makita hdrsize = sizeof(struct gre_base_hdr);
611fcb6aa86SToshiaki Makita break;
612fcb6aa86SToshiaki Makita #endif
613fcb6aa86SToshiaki Makita default:
61486360030SDan Carpenter return false;
615fcb6aa86SToshiaki Makita }
61646475bb2SPaul Blakey
61746475bb2SPaul Blakey if (ip6h->hop_limit <= 1)
61846475bb2SPaul Blakey return false;
61946475bb2SPaul Blakey
620fcb6aa86SToshiaki Makita if (!pskb_network_may_pull(skb, thoff + hdrsize))
62146475bb2SPaul Blakey return false;
62246475bb2SPaul Blakey
623fcb6aa86SToshiaki Makita switch (nexthdr) {
624fcb6aa86SToshiaki Makita case IPPROTO_TCP:
62507ac9d16SPaul Blakey *tcph = (void *)(skb_network_header(skb) + thoff);
626fcb6aa86SToshiaki Makita fallthrough;
627fcb6aa86SToshiaki Makita case IPPROTO_UDP:
62807ac9d16SPaul Blakey ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
62946475bb2SPaul Blakey tuple->src_port = ports->source;
63046475bb2SPaul Blakey tuple->dst_port = ports->dest;
631fcb6aa86SToshiaki Makita break;
632fcb6aa86SToshiaki Makita case IPPROTO_GRE: {
633fcb6aa86SToshiaki Makita struct gre_base_hdr *greh;
634fcb6aa86SToshiaki Makita
635fcb6aa86SToshiaki Makita greh = (struct gre_base_hdr *)(skb_network_header(skb) + thoff);
636fcb6aa86SToshiaki Makita if ((greh->flags & GRE_VERSION) != GRE_VERSION_0)
637fcb6aa86SToshiaki Makita return false;
638fcb6aa86SToshiaki Makita break;
639fcb6aa86SToshiaki Makita }
640fcb6aa86SToshiaki Makita }
641fcb6aa86SToshiaki Makita
642fcb6aa86SToshiaki Makita ip6h = ipv6_hdr(skb);
643fcb6aa86SToshiaki Makita
644fcb6aa86SToshiaki Makita tuple->src_v6 = ip6h->saddr;
645fcb6aa86SToshiaki Makita tuple->dst_v6 = ip6h->daddr;
64646475bb2SPaul Blakey tuple->l3proto = AF_INET6;
647fcb6aa86SToshiaki Makita tuple->l4proto = nexthdr;
64846475bb2SPaul Blakey
64946475bb2SPaul Blakey return true;
65046475bb2SPaul Blakey }
65146475bb2SPaul Blakey
tcf_ct_flow_table_lookup(struct tcf_ct_params * p,struct sk_buff * skb,u8 family)65246475bb2SPaul Blakey static bool tcf_ct_flow_table_lookup(struct tcf_ct_params *p,
65346475bb2SPaul Blakey struct sk_buff *skb,
65446475bb2SPaul Blakey u8 family)
65546475bb2SPaul Blakey {
65646475bb2SPaul Blakey struct nf_flowtable *nf_ft = &p->ct_ft->nf_ft;
65746475bb2SPaul Blakey struct flow_offload_tuple_rhash *tuplehash;
65846475bb2SPaul Blakey struct flow_offload_tuple tuple = {};
65946475bb2SPaul Blakey enum ip_conntrack_info ctinfo;
66007ac9d16SPaul Blakey struct tcphdr *tcph = NULL;
66141f2c7c3SPaul Blakey bool force_refresh = false;
66246475bb2SPaul Blakey struct flow_offload *flow;
66346475bb2SPaul Blakey struct nf_conn *ct;
66446475bb2SPaul Blakey u8 dir;
66546475bb2SPaul Blakey
66646475bb2SPaul Blakey switch (family) {
66746475bb2SPaul Blakey case NFPROTO_IPV4:
66807ac9d16SPaul Blakey if (!tcf_ct_flow_table_fill_tuple_ipv4(skb, &tuple, &tcph))
66946475bb2SPaul Blakey return false;
67046475bb2SPaul Blakey break;
67146475bb2SPaul Blakey case NFPROTO_IPV6:
67207ac9d16SPaul Blakey if (!tcf_ct_flow_table_fill_tuple_ipv6(skb, &tuple, &tcph))
67346475bb2SPaul Blakey return false;
67446475bb2SPaul Blakey break;
67546475bb2SPaul Blakey default:
67646475bb2SPaul Blakey return false;
67746475bb2SPaul Blakey }
67846475bb2SPaul Blakey
67946475bb2SPaul Blakey tuplehash = flow_offload_lookup(nf_ft, &tuple);
68046475bb2SPaul Blakey if (!tuplehash)
68146475bb2SPaul Blakey return false;
68246475bb2SPaul Blakey
68346475bb2SPaul Blakey dir = tuplehash->tuple.dir;
68446475bb2SPaul Blakey flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
68546475bb2SPaul Blakey ct = flow->ct;
68646475bb2SPaul Blakey
6876a9bad00SVlad Buslov if (dir == FLOW_OFFLOAD_DIR_REPLY &&
6886a9bad00SVlad Buslov !test_bit(NF_FLOW_HW_BIDIRECTIONAL, &flow->flags)) {
6896a9bad00SVlad Buslov /* Only offload reply direction after connection became
6906a9bad00SVlad Buslov * assured.
6916a9bad00SVlad Buslov */
6926a9bad00SVlad Buslov if (test_bit(IPS_ASSURED_BIT, &ct->status))
6936a9bad00SVlad Buslov set_bit(NF_FLOW_HW_BIDIRECTIONAL, &flow->flags);
6946a9bad00SVlad Buslov else if (test_bit(NF_FLOW_HW_ESTABLISHED, &flow->flags))
6956a9bad00SVlad Buslov /* If flow_table flow has already been updated to the
6966a9bad00SVlad Buslov * established state, then don't refresh.
6976a9bad00SVlad Buslov */
6986a9bad00SVlad Buslov return false;
69941f2c7c3SPaul Blakey force_refresh = true;
7006a9bad00SVlad Buslov }
7016a9bad00SVlad Buslov
70207ac9d16SPaul Blakey if (tcph && (unlikely(tcph->fin || tcph->rst))) {
70307ac9d16SPaul Blakey flow_offload_teardown(flow);
70407ac9d16SPaul Blakey return false;
70507ac9d16SPaul Blakey }
70607ac9d16SPaul Blakey
7076a9bad00SVlad Buslov if (dir == FLOW_OFFLOAD_DIR_ORIGINAL)
7086a9bad00SVlad Buslov ctinfo = test_bit(IPS_SEEN_REPLY_BIT, &ct->status) ?
7096a9bad00SVlad Buslov IP_CT_ESTABLISHED : IP_CT_NEW;
7106a9bad00SVlad Buslov else
7116a9bad00SVlad Buslov ctinfo = IP_CT_ESTABLISHED_REPLY;
71246475bb2SPaul Blakey
713d5a116dbSVlad Buslov nf_conn_act_ct_ext_fill(skb, ct, ctinfo);
714d5a116dbSVlad Buslov tcf_ct_flow_ct_ext_ifidx_update(flow);
71541f2c7c3SPaul Blakey flow_offload_refresh(nf_ft, flow, force_refresh);
71641f2c7c3SPaul Blakey if (!test_bit(IPS_ASSURED_BIT, &ct->status)) {
71741f2c7c3SPaul Blakey /* Process this flow in SW to allow promoting to ASSURED */
71841f2c7c3SPaul Blakey return false;
71941f2c7c3SPaul Blakey }
72041f2c7c3SPaul Blakey
72146475bb2SPaul Blakey nf_conntrack_get(&ct->ct_general);
72246475bb2SPaul Blakey nf_ct_set(skb, ct, ctinfo);
7233567e233SMarcelo Ricardo Leitner if (nf_ft->flags & NF_FLOWTABLE_COUNTER)
724beb97d3aSwenxu nf_ct_acct_update(ct, dir, skb->len);
72546475bb2SPaul Blakey
72646475bb2SPaul Blakey return true;
72746475bb2SPaul Blakey }
72846475bb2SPaul Blakey
tcf_ct_flow_tables_init(void)729c34b961aSPaul Blakey static int tcf_ct_flow_tables_init(void)
730c34b961aSPaul Blakey {
731c34b961aSPaul Blakey return rhashtable_init(&zones_ht, &zones_params);
732c34b961aSPaul Blakey }
733c34b961aSPaul Blakey
tcf_ct_flow_tables_uninit(void)734c34b961aSPaul Blakey static void tcf_ct_flow_tables_uninit(void)
735c34b961aSPaul Blakey {
736c34b961aSPaul Blakey rhashtable_destroy(&zones_ht);
737c34b961aSPaul Blakey }
738c34b961aSPaul Blakey
739b57dc7c1SPaul Blakey static struct tc_action_ops act_ct_ops;
740b57dc7c1SPaul Blakey
741b57dc7c1SPaul Blakey struct tc_ct_action_net {
742b57dc7c1SPaul Blakey struct tc_action_net tn; /* Must be first */
743b57dc7c1SPaul Blakey bool labels;
744b57dc7c1SPaul Blakey };
745b57dc7c1SPaul Blakey
746b57dc7c1SPaul Blakey /* Determine whether skb->_nfct is equal to the result of conntrack lookup. */
tcf_ct_skb_nfct_cached(struct net * net,struct sk_buff * skb,struct tcf_ct_params * p)747b57dc7c1SPaul Blakey static bool tcf_ct_skb_nfct_cached(struct net *net, struct sk_buff *skb,
748a21b06e7SXin Long struct tcf_ct_params *p)
749b57dc7c1SPaul Blakey {
750b57dc7c1SPaul Blakey enum ip_conntrack_info ctinfo;
751b57dc7c1SPaul Blakey struct nf_conn *ct;
752b57dc7c1SPaul Blakey
753b57dc7c1SPaul Blakey ct = nf_ct_get(skb, &ctinfo);
754b57dc7c1SPaul Blakey if (!ct)
755b57dc7c1SPaul Blakey return false;
756b57dc7c1SPaul Blakey if (!net_eq(net, read_pnet(&ct->ct_net)))
757bcb74e13SMarcelo Ricardo Leitner goto drop_ct;
758a21b06e7SXin Long if (nf_ct_zone(ct)->id != p->zone)
759bcb74e13SMarcelo Ricardo Leitner goto drop_ct;
760a21b06e7SXin Long if (p->helper) {
761a21b06e7SXin Long struct nf_conn_help *help;
762a21b06e7SXin Long
763a21b06e7SXin Long help = nf_ct_ext_find(ct, NF_CT_EXT_HELPER);
764a21b06e7SXin Long if (help && rcu_access_pointer(help->helper) != p->helper)
765a21b06e7SXin Long goto drop_ct;
766a21b06e7SXin Long }
767b57dc7c1SPaul Blakey
768b57dc7c1SPaul Blakey /* Force conntrack entry direction. */
769a21b06e7SXin Long if ((p->ct_action & TCA_CT_ACT_FORCE) &&
770a21b06e7SXin Long CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
771b57dc7c1SPaul Blakey if (nf_ct_is_confirmed(ct))
772b57dc7c1SPaul Blakey nf_ct_kill(ct);
773b57dc7c1SPaul Blakey
774bcb74e13SMarcelo Ricardo Leitner goto drop_ct;
775bcb74e13SMarcelo Ricardo Leitner }
776bcb74e13SMarcelo Ricardo Leitner
777bcb74e13SMarcelo Ricardo Leitner return true;
778bcb74e13SMarcelo Ricardo Leitner
779bcb74e13SMarcelo Ricardo Leitner drop_ct:
780408bdcfcSFlorian Westphal nf_ct_put(ct);
781b57dc7c1SPaul Blakey nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
782b57dc7c1SPaul Blakey
783b57dc7c1SPaul Blakey return false;
784b57dc7c1SPaul Blakey }
785b57dc7c1SPaul Blakey
tcf_ct_skb_nf_family(struct sk_buff * skb)786b57dc7c1SPaul Blakey static u8 tcf_ct_skb_nf_family(struct sk_buff *skb)
787b57dc7c1SPaul Blakey {
788b57dc7c1SPaul Blakey u8 family = NFPROTO_UNSPEC;
789b57dc7c1SPaul Blakey
790d7bf2ebeSToke Høiland-Jørgensen switch (skb_protocol(skb, true)) {
791b57dc7c1SPaul Blakey case htons(ETH_P_IP):
792b57dc7c1SPaul Blakey family = NFPROTO_IPV4;
793b57dc7c1SPaul Blakey break;
794b57dc7c1SPaul Blakey case htons(ETH_P_IPV6):
795b57dc7c1SPaul Blakey family = NFPROTO_IPV6;
796b57dc7c1SPaul Blakey break;
797b57dc7c1SPaul Blakey default:
798b57dc7c1SPaul Blakey break;
799b57dc7c1SPaul Blakey }
800b57dc7c1SPaul Blakey
801b57dc7c1SPaul Blakey return family;
802b57dc7c1SPaul Blakey }
803b57dc7c1SPaul Blakey
tcf_ct_ipv4_is_fragment(struct sk_buff * skb,bool * frag)804b57dc7c1SPaul Blakey static int tcf_ct_ipv4_is_fragment(struct sk_buff *skb, bool *frag)
805b57dc7c1SPaul Blakey {
806b57dc7c1SPaul Blakey unsigned int len;
807b57dc7c1SPaul Blakey
808b57dc7c1SPaul Blakey len = skb_network_offset(skb) + sizeof(struct iphdr);
809b57dc7c1SPaul Blakey if (unlikely(skb->len < len))
810b57dc7c1SPaul Blakey return -EINVAL;
811b57dc7c1SPaul Blakey if (unlikely(!pskb_may_pull(skb, len)))
812b57dc7c1SPaul Blakey return -ENOMEM;
813b57dc7c1SPaul Blakey
814b57dc7c1SPaul Blakey *frag = ip_is_fragment(ip_hdr(skb));
815b57dc7c1SPaul Blakey return 0;
816b57dc7c1SPaul Blakey }
817b57dc7c1SPaul Blakey
tcf_ct_ipv6_is_fragment(struct sk_buff * skb,bool * frag)818b57dc7c1SPaul Blakey static int tcf_ct_ipv6_is_fragment(struct sk_buff *skb, bool *frag)
819b57dc7c1SPaul Blakey {
820b57dc7c1SPaul Blakey unsigned int flags = 0, len, payload_ofs = 0;
821b57dc7c1SPaul Blakey unsigned short frag_off;
822b57dc7c1SPaul Blakey int nexthdr;
823b57dc7c1SPaul Blakey
824b57dc7c1SPaul Blakey len = skb_network_offset(skb) + sizeof(struct ipv6hdr);
825b57dc7c1SPaul Blakey if (unlikely(skb->len < len))
826b57dc7c1SPaul Blakey return -EINVAL;
827b57dc7c1SPaul Blakey if (unlikely(!pskb_may_pull(skb, len)))
828b57dc7c1SPaul Blakey return -ENOMEM;
829b57dc7c1SPaul Blakey
830b57dc7c1SPaul Blakey nexthdr = ipv6_find_hdr(skb, &payload_ofs, -1, &frag_off, &flags);
831b57dc7c1SPaul Blakey if (unlikely(nexthdr < 0))
832b57dc7c1SPaul Blakey return -EPROTO;
833b57dc7c1SPaul Blakey
834b57dc7c1SPaul Blakey *frag = flags & IP6_FH_F_FRAG;
835b57dc7c1SPaul Blakey return 0;
836b57dc7c1SPaul Blakey }
837b57dc7c1SPaul Blakey
tcf_ct_handle_fragments(struct net * net,struct sk_buff * skb,u8 family,u16 zone,bool * defrag)838b57dc7c1SPaul Blakey static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
839ae372cb1Swenxu u8 family, u16 zone, bool *defrag)
840b57dc7c1SPaul Blakey {
841b57dc7c1SPaul Blakey enum ip_conntrack_info ctinfo;
842b57dc7c1SPaul Blakey struct nf_conn *ct;
843b57dc7c1SPaul Blakey int err = 0;
844b57dc7c1SPaul Blakey bool frag;
8450785407eSXin Long u8 proto;
846ec624fe7SPaul Blakey u16 mru;
847b57dc7c1SPaul Blakey
848b57dc7c1SPaul Blakey /* Previously seen (loopback)? Ignore. */
849b57dc7c1SPaul Blakey ct = nf_ct_get(skb, &ctinfo);
850b57dc7c1SPaul Blakey if ((ct && !nf_ct_is_template(ct)) || ctinfo == IP_CT_UNTRACKED)
851b57dc7c1SPaul Blakey return 0;
852b57dc7c1SPaul Blakey
853b57dc7c1SPaul Blakey if (family == NFPROTO_IPV4)
854b57dc7c1SPaul Blakey err = tcf_ct_ipv4_is_fragment(skb, &frag);
855b57dc7c1SPaul Blakey else
856b57dc7c1SPaul Blakey err = tcf_ct_ipv6_is_fragment(skb, &frag);
857b57dc7c1SPaul Blakey if (err || !frag)
858b57dc7c1SPaul Blakey return err;
859b57dc7c1SPaul Blakey
8600785407eSXin Long err = nf_ct_handle_fragments(net, skb, zone, family, &proto, &mru);
861558d95e7SXin Long if (err)
862eda814b9SAlaa Hleihel return err;
863ae372cb1Swenxu
864ae372cb1Swenxu *defrag = true;
865ec624fe7SPaul Blakey tc_skb_cb(skb)->mru = mru;
866b57dc7c1SPaul Blakey
867558d95e7SXin Long return 0;
868b57dc7c1SPaul Blakey }
869b57dc7c1SPaul Blakey
tcf_ct_params_free(struct tcf_ct_params * params)87019138941SXin Long static void tcf_ct_params_free(struct tcf_ct_params *params)
871b57dc7c1SPaul Blakey {
872a21b06e7SXin Long if (params->helper) {
873a21b06e7SXin Long #if IS_ENABLED(CONFIG_NF_NAT)
874a21b06e7SXin Long if (params->ct_action & TCA_CT_ACT_NAT)
875a21b06e7SXin Long nf_nat_helper_put(params->helper);
876a21b06e7SXin Long #endif
877a21b06e7SXin Long nf_conntrack_helper_put(params->helper);
878a21b06e7SXin Long }
87919138941SXin Long if (params->ct_ft)
88019138941SXin Long tcf_ct_flow_table_put(params->ct_ft);
881b57dc7c1SPaul Blakey if (params->tmpl)
882408bdcfcSFlorian Westphal nf_ct_put(params->tmpl);
883b57dc7c1SPaul Blakey kfree(params);
884b57dc7c1SPaul Blakey }
885b57dc7c1SPaul Blakey
tcf_ct_params_free_rcu(struct rcu_head * head)88619138941SXin Long static void tcf_ct_params_free_rcu(struct rcu_head *head)
88719138941SXin Long {
88819138941SXin Long struct tcf_ct_params *params;
88919138941SXin Long
89019138941SXin Long params = container_of(head, struct tcf_ct_params, rcu);
89119138941SXin Long tcf_ct_params_free(params);
89219138941SXin Long }
89319138941SXin Long
tcf_ct_act_set_mark(struct nf_conn * ct,u32 mark,u32 mask)894b57dc7c1SPaul Blakey static void tcf_ct_act_set_mark(struct nf_conn *ct, u32 mark, u32 mask)
895b57dc7c1SPaul Blakey {
896b57dc7c1SPaul Blakey #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
897b57dc7c1SPaul Blakey u32 new_mark;
898b57dc7c1SPaul Blakey
899b57dc7c1SPaul Blakey if (!mask)
900b57dc7c1SPaul Blakey return;
901b57dc7c1SPaul Blakey
90252d1aa8bSDaniel Xu new_mark = mark | (READ_ONCE(ct->mark) & ~(mask));
90352d1aa8bSDaniel Xu if (READ_ONCE(ct->mark) != new_mark) {
90452d1aa8bSDaniel Xu WRITE_ONCE(ct->mark, new_mark);
905b57dc7c1SPaul Blakey if (nf_ct_is_confirmed(ct))
906b57dc7c1SPaul Blakey nf_conntrack_event_cache(IPCT_MARK, ct);
907b57dc7c1SPaul Blakey }
908b57dc7c1SPaul Blakey #endif
909b57dc7c1SPaul Blakey }
910b57dc7c1SPaul Blakey
tcf_ct_act_set_labels(struct nf_conn * ct,u32 * labels,u32 * labels_m)911b57dc7c1SPaul Blakey static void tcf_ct_act_set_labels(struct nf_conn *ct,
912b57dc7c1SPaul Blakey u32 *labels,
913b57dc7c1SPaul Blakey u32 *labels_m)
914b57dc7c1SPaul Blakey {
915b57dc7c1SPaul Blakey #if IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)
916c593642cSPankaj Bharadiya size_t labels_sz = sizeof_field(struct tcf_ct_params, labels);
917b57dc7c1SPaul Blakey
918b57dc7c1SPaul Blakey if (!memchr_inv(labels_m, 0, labels_sz))
919b57dc7c1SPaul Blakey return;
920b57dc7c1SPaul Blakey
921b57dc7c1SPaul Blakey nf_connlabels_replace(ct, labels, labels_m, 4);
922b57dc7c1SPaul Blakey #endif
923b57dc7c1SPaul Blakey }
924b57dc7c1SPaul Blakey
tcf_ct_act_nat(struct sk_buff * skb,struct nf_conn * ct,enum ip_conntrack_info ctinfo,int ct_action,struct nf_nat_range2 * range,bool commit)925b57dc7c1SPaul Blakey static int tcf_ct_act_nat(struct sk_buff *skb,
926b57dc7c1SPaul Blakey struct nf_conn *ct,
927b57dc7c1SPaul Blakey enum ip_conntrack_info ctinfo,
928b57dc7c1SPaul Blakey int ct_action,
929b57dc7c1SPaul Blakey struct nf_nat_range2 *range,
930b57dc7c1SPaul Blakey bool commit)
931b57dc7c1SPaul Blakey {
932b57dc7c1SPaul Blakey #if IS_ENABLED(CONFIG_NF_NAT)
933ebddb140SXin Long int err, action = 0;
934b57dc7c1SPaul Blakey
935b57dc7c1SPaul Blakey if (!(ct_action & TCA_CT_ACT_NAT))
936b57dc7c1SPaul Blakey return NF_ACCEPT;
937ebddb140SXin Long if (ct_action & TCA_CT_ACT_NAT_SRC)
938ebddb140SXin Long action |= BIT(NF_NAT_MANIP_SRC);
939ebddb140SXin Long if (ct_action & TCA_CT_ACT_NAT_DST)
940ebddb140SXin Long action |= BIT(NF_NAT_MANIP_DST);
941b57dc7c1SPaul Blakey
942ebddb140SXin Long err = nf_ct_nat(skb, ct, ctinfo, &action, range, commit);
943b57dc7c1SPaul Blakey
944ebddb140SXin Long if (action & BIT(NF_NAT_MANIP_SRC))
945ebddb140SXin Long tc_skb_cb(skb)->post_ct_snat = 1;
946ebddb140SXin Long if (action & BIT(NF_NAT_MANIP_DST))
947ebddb140SXin Long tc_skb_cb(skb)->post_ct_dnat = 1;
948b57dc7c1SPaul Blakey
94995219afbSAaron Conole return err;
950b57dc7c1SPaul Blakey #else
951b57dc7c1SPaul Blakey return NF_ACCEPT;
952b57dc7c1SPaul Blakey #endif
953b57dc7c1SPaul Blakey }
954b57dc7c1SPaul Blakey
tcf_ct_act(struct sk_buff * skb,const struct tc_action * a,struct tcf_result * res)955871cf386SPedro Tammela TC_INDIRECT_SCOPE int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
956b57dc7c1SPaul Blakey struct tcf_result *res)
957b57dc7c1SPaul Blakey {
958b57dc7c1SPaul Blakey struct net *net = dev_net(skb->dev);
959b57dc7c1SPaul Blakey enum ip_conntrack_info ctinfo;
960b57dc7c1SPaul Blakey struct tcf_ct *c = to_ct(a);
961b57dc7c1SPaul Blakey struct nf_conn *tmpl = NULL;
962b57dc7c1SPaul Blakey struct nf_hook_state state;
963a21b06e7SXin Long bool cached, commit, clear;
964b57dc7c1SPaul Blakey int nh_ofs, err, retval;
965b57dc7c1SPaul Blakey struct tcf_ct_params *p;
966a21b06e7SXin Long bool add_helper = false;
96746475bb2SPaul Blakey bool skip_add = false;
968ae372cb1Swenxu bool defrag = false;
969b57dc7c1SPaul Blakey struct nf_conn *ct;
970b57dc7c1SPaul Blakey u8 family;
971b57dc7c1SPaul Blakey
972b57dc7c1SPaul Blakey p = rcu_dereference_bh(c->params);
973b57dc7c1SPaul Blakey
974b57dc7c1SPaul Blakey retval = READ_ONCE(c->tcf_action);
975b57dc7c1SPaul Blakey commit = p->ct_action & TCA_CT_ACT_COMMIT;
976b57dc7c1SPaul Blakey clear = p->ct_action & TCA_CT_ACT_CLEAR;
977b57dc7c1SPaul Blakey tmpl = p->tmpl;
978b57dc7c1SPaul Blakey
9798367b3abSwenxu tcf_lastuse_update(&c->tcf_tm);
9802dc4e9e8SPaul Blakey tcf_action_update_bstats(&c->common, skb);
9818367b3abSwenxu
982b57dc7c1SPaul Blakey if (clear) {
983ec624fe7SPaul Blakey tc_skb_cb(skb)->post_ct = false;
984b57dc7c1SPaul Blakey ct = nf_ct_get(skb, &ctinfo);
985b57dc7c1SPaul Blakey if (ct) {
986408bdcfcSFlorian Westphal nf_ct_put(ct);
987b57dc7c1SPaul Blakey nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
988b57dc7c1SPaul Blakey }
989b57dc7c1SPaul Blakey
9908ca1b090SMarcelo Ricardo Leitner goto out_clear;
991b57dc7c1SPaul Blakey }
992b57dc7c1SPaul Blakey
993b57dc7c1SPaul Blakey family = tcf_ct_skb_nf_family(skb);
994b57dc7c1SPaul Blakey if (family == NFPROTO_UNSPEC)
995b57dc7c1SPaul Blakey goto drop;
996b57dc7c1SPaul Blakey
997b57dc7c1SPaul Blakey /* The conntrack module expects to be working at L3.
998b57dc7c1SPaul Blakey * We also try to pull the IPv4/6 header to linear area
999b57dc7c1SPaul Blakey */
1000b57dc7c1SPaul Blakey nh_ofs = skb_network_offset(skb);
1001b57dc7c1SPaul Blakey skb_pull_rcsum(skb, nh_ofs);
1002ae372cb1Swenxu err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag);
1003b57dc7c1SPaul Blakey if (err)
100473f7da5fSTao Liu goto out_frag;
1005b57dc7c1SPaul Blakey
100667fc5d7fSXin Long err = nf_ct_skb_network_trim(skb, family);
1007b57dc7c1SPaul Blakey if (err)
1008b57dc7c1SPaul Blakey goto drop;
1009b57dc7c1SPaul Blakey
1010b57dc7c1SPaul Blakey /* If we are recirculating packets to match on ct fields and
1011b57dc7c1SPaul Blakey * committing with a separate ct action, then we don't need to
1012b57dc7c1SPaul Blakey * actually run the packet through conntrack twice unless it's for a
1013b57dc7c1SPaul Blakey * different zone.
1014b57dc7c1SPaul Blakey */
1015a21b06e7SXin Long cached = tcf_ct_skb_nfct_cached(net, skb, p);
1016b57dc7c1SPaul Blakey if (!cached) {
10170cc254e5SPaul Blakey if (tcf_ct_flow_table_lookup(p, skb, family)) {
101846475bb2SPaul Blakey skip_add = true;
101946475bb2SPaul Blakey goto do_nat;
102046475bb2SPaul Blakey }
102146475bb2SPaul Blakey
1022b57dc7c1SPaul Blakey /* Associate skb with specified zone. */
1023b57dc7c1SPaul Blakey if (tmpl) {
1024b57dc7c1SPaul Blakey nf_conntrack_put(skb_nfct(skb));
1025b57dc7c1SPaul Blakey nf_conntrack_get(&tmpl->ct_general);
1026b57dc7c1SPaul Blakey nf_ct_set(skb, tmpl, IP_CT_NEW);
1027b57dc7c1SPaul Blakey }
1028b57dc7c1SPaul Blakey
1029b57dc7c1SPaul Blakey state.hook = NF_INET_PRE_ROUTING;
1030b57dc7c1SPaul Blakey state.net = net;
1031b57dc7c1SPaul Blakey state.pf = family;
1032b57dc7c1SPaul Blakey err = nf_conntrack_in(skb, &state);
1033b57dc7c1SPaul Blakey if (err != NF_ACCEPT)
1034b57dc7c1SPaul Blakey goto out_push;
1035b57dc7c1SPaul Blakey }
1036b57dc7c1SPaul Blakey
103746475bb2SPaul Blakey do_nat:
1038b57dc7c1SPaul Blakey ct = nf_ct_get(skb, &ctinfo);
1039b57dc7c1SPaul Blakey if (!ct)
1040b57dc7c1SPaul Blakey goto out_push;
1041b57dc7c1SPaul Blakey nf_ct_deliver_cached_events(ct);
10429795ded7SPaul Blakey nf_conn_act_ct_ext_fill(skb, ct, ctinfo);
1043b57dc7c1SPaul Blakey
1044b57dc7c1SPaul Blakey err = tcf_ct_act_nat(skb, ct, ctinfo, p->ct_action, &p->range, commit);
1045b57dc7c1SPaul Blakey if (err != NF_ACCEPT)
1046b57dc7c1SPaul Blakey goto drop;
1047b57dc7c1SPaul Blakey
1048a21b06e7SXin Long if (!nf_ct_is_confirmed(ct) && commit && p->helper && !nfct_help(ct)) {
1049a21b06e7SXin Long err = __nf_ct_try_assign_helper(ct, p->tmpl, GFP_ATOMIC);
1050a21b06e7SXin Long if (err)
1051a21b06e7SXin Long goto drop;
1052a21b06e7SXin Long add_helper = true;
1053a21b06e7SXin Long if (p->ct_action & TCA_CT_ACT_NAT && !nfct_seqadj(ct)) {
1054a21b06e7SXin Long if (!nfct_seqadj_ext_add(ct))
1055a21b06e7SXin Long goto drop;
1056a21b06e7SXin Long }
1057a21b06e7SXin Long }
1058a21b06e7SXin Long
1059a21b06e7SXin Long if (nf_ct_is_confirmed(ct) ? ((!cached && !skip_add) || add_helper) : commit) {
1060a21b06e7SXin Long if (nf_ct_helper(skb, ct, ctinfo, family) != NF_ACCEPT)
1061a21b06e7SXin Long goto drop;
1062a21b06e7SXin Long }
1063a21b06e7SXin Long
1064b57dc7c1SPaul Blakey if (commit) {
1065b57dc7c1SPaul Blakey tcf_ct_act_set_mark(ct, p->mark, p->mark_mask);
1066b57dc7c1SPaul Blakey tcf_ct_act_set_labels(ct, p->labels, p->labels_mask);
1067b57dc7c1SPaul Blakey
10689795ded7SPaul Blakey if (!nf_ct_is_confirmed(ct))
1069d5a116dbSVlad Buslov nf_conn_act_ct_ext_add(skb, ct, ctinfo);
10709795ded7SPaul Blakey
1071b57dc7c1SPaul Blakey /* This will take care of sending queued events
1072b57dc7c1SPaul Blakey * even if the connection is already confirmed.
1073b57dc7c1SPaul Blakey */
10748955b90cSwenxu if (nf_conntrack_confirm(skb) != NF_ACCEPT)
10758955b90cSwenxu goto drop;
107646475bb2SPaul Blakey }
107764ff70b8SPaul Blakey
10780cc254e5SPaul Blakey if (!skip_add)
10790cc254e5SPaul Blakey tcf_ct_flow_table_process_conn(p->ct_ft, ct, ctinfo);
10800cc254e5SPaul Blakey
1081b57dc7c1SPaul Blakey out_push:
1082b57dc7c1SPaul Blakey skb_push_rcsum(skb, nh_ofs);
1083b57dc7c1SPaul Blakey
1084ec624fe7SPaul Blakey tc_skb_cb(skb)->post_ct = true;
108538495958SPaul Blakey tc_skb_cb(skb)->zone = p->zone;
10868ca1b090SMarcelo Ricardo Leitner out_clear:
1087ae372cb1Swenxu if (defrag)
1088ae372cb1Swenxu qdisc_skb_cb(skb)->pkt_len = skb->len;
1089b57dc7c1SPaul Blakey return retval;
1090b57dc7c1SPaul Blakey
109173f7da5fSTao Liu out_frag:
109273f7da5fSTao Liu if (err != -EINPROGRESS)
109373f7da5fSTao Liu tcf_action_inc_drop_qstats(&c->common);
109473f7da5fSTao Liu return TC_ACT_CONSUMED;
109573f7da5fSTao Liu
1096b57dc7c1SPaul Blakey drop:
109726b537a8SVlad Buslov tcf_action_inc_drop_qstats(&c->common);
1098b57dc7c1SPaul Blakey return TC_ACT_SHOT;
1099b57dc7c1SPaul Blakey }
1100b57dc7c1SPaul Blakey
1101b57dc7c1SPaul Blakey static const struct nla_policy ct_policy[TCA_CT_MAX + 1] = {
1102b57dc7c1SPaul Blakey [TCA_CT_ACTION] = { .type = NLA_U16 },
11038140860cSJohannes Berg [TCA_CT_PARMS] = NLA_POLICY_EXACT_LEN(sizeof(struct tc_ct)),
1104b57dc7c1SPaul Blakey [TCA_CT_ZONE] = { .type = NLA_U16 },
1105b57dc7c1SPaul Blakey [TCA_CT_MARK] = { .type = NLA_U32 },
1106b57dc7c1SPaul Blakey [TCA_CT_MARK_MASK] = { .type = NLA_U32 },
1107b57dc7c1SPaul Blakey [TCA_CT_LABELS] = { .type = NLA_BINARY,
1108b57dc7c1SPaul Blakey .len = 128 / BITS_PER_BYTE },
1109b57dc7c1SPaul Blakey [TCA_CT_LABELS_MASK] = { .type = NLA_BINARY,
1110b57dc7c1SPaul Blakey .len = 128 / BITS_PER_BYTE },
1111b57dc7c1SPaul Blakey [TCA_CT_NAT_IPV4_MIN] = { .type = NLA_U32 },
1112b57dc7c1SPaul Blakey [TCA_CT_NAT_IPV4_MAX] = { .type = NLA_U32 },
11138140860cSJohannes Berg [TCA_CT_NAT_IPV6_MIN] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
11148140860cSJohannes Berg [TCA_CT_NAT_IPV6_MAX] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
1115b57dc7c1SPaul Blakey [TCA_CT_NAT_PORT_MIN] = { .type = NLA_U16 },
1116b57dc7c1SPaul Blakey [TCA_CT_NAT_PORT_MAX] = { .type = NLA_U16 },
1117a21b06e7SXin Long [TCA_CT_HELPER_NAME] = { .type = NLA_STRING, .len = NF_CT_HELPER_NAME_LEN },
1118a21b06e7SXin Long [TCA_CT_HELPER_FAMILY] = { .type = NLA_U8 },
1119a21b06e7SXin Long [TCA_CT_HELPER_PROTO] = { .type = NLA_U8 },
1120b57dc7c1SPaul Blakey };
1121b57dc7c1SPaul Blakey
tcf_ct_fill_params_nat(struct tcf_ct_params * p,struct tc_ct * parm,struct nlattr ** tb,struct netlink_ext_ack * extack)1122b57dc7c1SPaul Blakey static int tcf_ct_fill_params_nat(struct tcf_ct_params *p,
1123b57dc7c1SPaul Blakey struct tc_ct *parm,
1124b57dc7c1SPaul Blakey struct nlattr **tb,
1125b57dc7c1SPaul Blakey struct netlink_ext_ack *extack)
1126b57dc7c1SPaul Blakey {
1127b57dc7c1SPaul Blakey struct nf_nat_range2 *range;
1128b57dc7c1SPaul Blakey
1129b57dc7c1SPaul Blakey if (!(p->ct_action & TCA_CT_ACT_NAT))
1130b57dc7c1SPaul Blakey return 0;
1131b57dc7c1SPaul Blakey
1132b57dc7c1SPaul Blakey if (!IS_ENABLED(CONFIG_NF_NAT)) {
1133b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Netfilter nat isn't enabled in kernel");
1134b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1135b57dc7c1SPaul Blakey }
1136b57dc7c1SPaul Blakey
1137b57dc7c1SPaul Blakey if (!(p->ct_action & (TCA_CT_ACT_NAT_SRC | TCA_CT_ACT_NAT_DST)))
1138b57dc7c1SPaul Blakey return 0;
1139b57dc7c1SPaul Blakey
1140b57dc7c1SPaul Blakey if ((p->ct_action & TCA_CT_ACT_NAT_SRC) &&
1141b57dc7c1SPaul Blakey (p->ct_action & TCA_CT_ACT_NAT_DST)) {
1142b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "dnat and snat can't be enabled at the same time");
1143b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1144b57dc7c1SPaul Blakey }
1145b57dc7c1SPaul Blakey
1146b57dc7c1SPaul Blakey range = &p->range;
1147b57dc7c1SPaul Blakey if (tb[TCA_CT_NAT_IPV4_MIN]) {
1148b57dc7c1SPaul Blakey struct nlattr *max_attr = tb[TCA_CT_NAT_IPV4_MAX];
1149b57dc7c1SPaul Blakey
1150b57dc7c1SPaul Blakey p->ipv4_range = true;
1151b57dc7c1SPaul Blakey range->flags |= NF_NAT_RANGE_MAP_IPS;
1152b57dc7c1SPaul Blakey range->min_addr.ip =
1153b57dc7c1SPaul Blakey nla_get_in_addr(tb[TCA_CT_NAT_IPV4_MIN]);
1154b57dc7c1SPaul Blakey
1155b57dc7c1SPaul Blakey range->max_addr.ip = max_attr ?
1156b57dc7c1SPaul Blakey nla_get_in_addr(max_attr) :
1157b57dc7c1SPaul Blakey range->min_addr.ip;
1158b57dc7c1SPaul Blakey } else if (tb[TCA_CT_NAT_IPV6_MIN]) {
1159b57dc7c1SPaul Blakey struct nlattr *max_attr = tb[TCA_CT_NAT_IPV6_MAX];
1160b57dc7c1SPaul Blakey
1161b57dc7c1SPaul Blakey p->ipv4_range = false;
1162b57dc7c1SPaul Blakey range->flags |= NF_NAT_RANGE_MAP_IPS;
1163b57dc7c1SPaul Blakey range->min_addr.in6 =
1164b57dc7c1SPaul Blakey nla_get_in6_addr(tb[TCA_CT_NAT_IPV6_MIN]);
1165b57dc7c1SPaul Blakey
1166b57dc7c1SPaul Blakey range->max_addr.in6 = max_attr ?
1167b57dc7c1SPaul Blakey nla_get_in6_addr(max_attr) :
1168b57dc7c1SPaul Blakey range->min_addr.in6;
1169b57dc7c1SPaul Blakey }
1170b57dc7c1SPaul Blakey
1171b57dc7c1SPaul Blakey if (tb[TCA_CT_NAT_PORT_MIN]) {
1172b57dc7c1SPaul Blakey range->flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
1173b57dc7c1SPaul Blakey range->min_proto.all = nla_get_be16(tb[TCA_CT_NAT_PORT_MIN]);
1174b57dc7c1SPaul Blakey
1175b57dc7c1SPaul Blakey range->max_proto.all = tb[TCA_CT_NAT_PORT_MAX] ?
1176b57dc7c1SPaul Blakey nla_get_be16(tb[TCA_CT_NAT_PORT_MAX]) :
1177b57dc7c1SPaul Blakey range->min_proto.all;
1178b57dc7c1SPaul Blakey }
1179b57dc7c1SPaul Blakey
1180b57dc7c1SPaul Blakey return 0;
1181b57dc7c1SPaul Blakey }
1182b57dc7c1SPaul Blakey
tcf_ct_set_key_val(struct nlattr ** tb,void * val,int val_type,void * mask,int mask_type,int len)1183b57dc7c1SPaul Blakey static void tcf_ct_set_key_val(struct nlattr **tb,
1184b57dc7c1SPaul Blakey void *val, int val_type,
1185b57dc7c1SPaul Blakey void *mask, int mask_type,
1186b57dc7c1SPaul Blakey int len)
1187b57dc7c1SPaul Blakey {
1188b57dc7c1SPaul Blakey if (!tb[val_type])
1189b57dc7c1SPaul Blakey return;
1190b57dc7c1SPaul Blakey nla_memcpy(val, tb[val_type], len);
1191b57dc7c1SPaul Blakey
1192b57dc7c1SPaul Blakey if (!mask)
1193b57dc7c1SPaul Blakey return;
1194b57dc7c1SPaul Blakey
1195b57dc7c1SPaul Blakey if (mask_type == TCA_CT_UNSPEC || !tb[mask_type])
1196b57dc7c1SPaul Blakey memset(mask, 0xff, len);
1197b57dc7c1SPaul Blakey else
1198b57dc7c1SPaul Blakey nla_memcpy(mask, tb[mask_type], len);
1199b57dc7c1SPaul Blakey }
1200b57dc7c1SPaul Blakey
tcf_ct_fill_params(struct net * net,struct tcf_ct_params * p,struct tc_ct * parm,struct nlattr ** tb,struct netlink_ext_ack * extack)1201b57dc7c1SPaul Blakey static int tcf_ct_fill_params(struct net *net,
1202b57dc7c1SPaul Blakey struct tcf_ct_params *p,
1203b57dc7c1SPaul Blakey struct tc_ct *parm,
1204b57dc7c1SPaul Blakey struct nlattr **tb,
1205b57dc7c1SPaul Blakey struct netlink_ext_ack *extack)
1206b57dc7c1SPaul Blakey {
1207acd0a7abSZhengchao Shao struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
1208b57dc7c1SPaul Blakey struct nf_conntrack_zone zone;
1209a21b06e7SXin Long int err, family, proto, len;
1210b57dc7c1SPaul Blakey struct nf_conn *tmpl;
1211a21b06e7SXin Long char *name;
1212b57dc7c1SPaul Blakey
1213b57dc7c1SPaul Blakey p->zone = NF_CT_DEFAULT_ZONE_ID;
1214b57dc7c1SPaul Blakey
1215b57dc7c1SPaul Blakey tcf_ct_set_key_val(tb,
1216b57dc7c1SPaul Blakey &p->ct_action, TCA_CT_ACTION,
1217b57dc7c1SPaul Blakey NULL, TCA_CT_UNSPEC,
1218b57dc7c1SPaul Blakey sizeof(p->ct_action));
1219b57dc7c1SPaul Blakey
1220b57dc7c1SPaul Blakey if (p->ct_action & TCA_CT_ACT_CLEAR)
1221b57dc7c1SPaul Blakey return 0;
1222b57dc7c1SPaul Blakey
1223b57dc7c1SPaul Blakey err = tcf_ct_fill_params_nat(p, parm, tb, extack);
1224b57dc7c1SPaul Blakey if (err)
1225b57dc7c1SPaul Blakey return err;
1226b57dc7c1SPaul Blakey
1227b57dc7c1SPaul Blakey if (tb[TCA_CT_MARK]) {
1228b57dc7c1SPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) {
1229b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Conntrack mark isn't enabled.");
1230b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1231b57dc7c1SPaul Blakey }
1232b57dc7c1SPaul Blakey tcf_ct_set_key_val(tb,
1233b57dc7c1SPaul Blakey &p->mark, TCA_CT_MARK,
1234b57dc7c1SPaul Blakey &p->mark_mask, TCA_CT_MARK_MASK,
1235b57dc7c1SPaul Blakey sizeof(p->mark));
1236b57dc7c1SPaul Blakey }
1237b57dc7c1SPaul Blakey
1238b57dc7c1SPaul Blakey if (tb[TCA_CT_LABELS]) {
1239b57dc7c1SPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) {
1240b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Conntrack labels isn't enabled.");
1241b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1242b57dc7c1SPaul Blakey }
1243b57dc7c1SPaul Blakey
1244b57dc7c1SPaul Blakey if (!tn->labels) {
1245b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Failed to set connlabel length");
1246b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1247b57dc7c1SPaul Blakey }
1248b57dc7c1SPaul Blakey tcf_ct_set_key_val(tb,
1249b57dc7c1SPaul Blakey p->labels, TCA_CT_LABELS,
1250b57dc7c1SPaul Blakey p->labels_mask, TCA_CT_LABELS_MASK,
1251b57dc7c1SPaul Blakey sizeof(p->labels));
1252b57dc7c1SPaul Blakey }
1253b57dc7c1SPaul Blakey
1254b57dc7c1SPaul Blakey if (tb[TCA_CT_ZONE]) {
1255b57dc7c1SPaul Blakey if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) {
1256b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Conntrack zones isn't enabled.");
1257b57dc7c1SPaul Blakey return -EOPNOTSUPP;
1258b57dc7c1SPaul Blakey }
1259b57dc7c1SPaul Blakey
1260b57dc7c1SPaul Blakey tcf_ct_set_key_val(tb,
1261b57dc7c1SPaul Blakey &p->zone, TCA_CT_ZONE,
1262b57dc7c1SPaul Blakey NULL, TCA_CT_UNSPEC,
1263b57dc7c1SPaul Blakey sizeof(p->zone));
1264b57dc7c1SPaul Blakey }
1265b57dc7c1SPaul Blakey
1266b57dc7c1SPaul Blakey nf_ct_zone_init(&zone, p->zone, NF_CT_DEFAULT_ZONE_DIR, 0);
1267b57dc7c1SPaul Blakey tmpl = nf_ct_tmpl_alloc(net, &zone, GFP_KERNEL);
1268b57dc7c1SPaul Blakey if (!tmpl) {
1269b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Failed to allocate conntrack template");
1270b57dc7c1SPaul Blakey return -ENOMEM;
1271b57dc7c1SPaul Blakey }
1272b57dc7c1SPaul Blakey p->tmpl = tmpl;
1273a21b06e7SXin Long if (tb[TCA_CT_HELPER_NAME]) {
1274a21b06e7SXin Long name = nla_data(tb[TCA_CT_HELPER_NAME]);
1275a21b06e7SXin Long len = nla_len(tb[TCA_CT_HELPER_NAME]);
1276a21b06e7SXin Long if (len > 16 || name[len - 1] != '\0') {
1277a21b06e7SXin Long NL_SET_ERR_MSG_MOD(extack, "Failed to parse helper name.");
1278a21b06e7SXin Long err = -EINVAL;
1279a21b06e7SXin Long goto err;
1280a21b06e7SXin Long }
1281a21b06e7SXin Long family = tb[TCA_CT_HELPER_FAMILY] ? nla_get_u8(tb[TCA_CT_HELPER_FAMILY]) : AF_INET;
1282a21b06e7SXin Long proto = tb[TCA_CT_HELPER_PROTO] ? nla_get_u8(tb[TCA_CT_HELPER_PROTO]) : IPPROTO_TCP;
1283a21b06e7SXin Long err = nf_ct_add_helper(tmpl, name, family, proto,
1284a21b06e7SXin Long p->ct_action & TCA_CT_ACT_NAT, &p->helper);
1285a21b06e7SXin Long if (err) {
1286a21b06e7SXin Long NL_SET_ERR_MSG_MOD(extack, "Failed to add helper");
1287a21b06e7SXin Long goto err;
1288a21b06e7SXin Long }
1289a21b06e7SXin Long }
1290b57dc7c1SPaul Blakey
129176622cedSXin Long if (p->ct_action & TCA_CT_ACT_COMMIT)
1292a21b06e7SXin Long __set_bit(IPS_CONFIRMED_BIT, &tmpl->status);
1293b57dc7c1SPaul Blakey return 0;
1294a21b06e7SXin Long err:
1295a21b06e7SXin Long nf_ct_put(p->tmpl);
1296a21b06e7SXin Long p->tmpl = NULL;
1297a21b06e7SXin Long return err;
1298b57dc7c1SPaul Blakey }
1299b57dc7c1SPaul Blakey
tcf_ct_init(struct net * net,struct nlattr * nla,struct nlattr * est,struct tc_action ** a,struct tcf_proto * tp,u32 flags,struct netlink_ext_ack * extack)1300b57dc7c1SPaul Blakey static int tcf_ct_init(struct net *net, struct nlattr *nla,
1301b57dc7c1SPaul Blakey struct nlattr *est, struct tc_action **a,
1302abbb0d33SVlad Buslov struct tcf_proto *tp, u32 flags,
1303b57dc7c1SPaul Blakey struct netlink_ext_ack *extack)
1304b57dc7c1SPaul Blakey {
1305acd0a7abSZhengchao Shao struct tc_action_net *tn = net_generic(net, act_ct_ops.net_id);
1306695176bfSCong Wang bool bind = flags & TCA_ACT_FLAGS_BIND;
1307b57dc7c1SPaul Blakey struct tcf_ct_params *params = NULL;
1308b57dc7c1SPaul Blakey struct nlattr *tb[TCA_CT_MAX + 1];
1309b57dc7c1SPaul Blakey struct tcf_chain *goto_ch = NULL;
1310b57dc7c1SPaul Blakey struct tc_ct *parm;
1311b57dc7c1SPaul Blakey struct tcf_ct *c;
1312b57dc7c1SPaul Blakey int err, res = 0;
13137be8ef2cSDmytro Linkin u32 index;
1314b57dc7c1SPaul Blakey
1315b57dc7c1SPaul Blakey if (!nla) {
1316b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Ct requires attributes to be passed");
1317b57dc7c1SPaul Blakey return -EINVAL;
1318b57dc7c1SPaul Blakey }
1319b57dc7c1SPaul Blakey
1320b57dc7c1SPaul Blakey err = nla_parse_nested(tb, TCA_CT_MAX, nla, ct_policy, extack);
1321b57dc7c1SPaul Blakey if (err < 0)
1322b57dc7c1SPaul Blakey return err;
1323b57dc7c1SPaul Blakey
1324b57dc7c1SPaul Blakey if (!tb[TCA_CT_PARMS]) {
1325b57dc7c1SPaul Blakey NL_SET_ERR_MSG_MOD(extack, "Missing required ct parameters");
1326b57dc7c1SPaul Blakey return -EINVAL;
1327b57dc7c1SPaul Blakey }
1328b57dc7c1SPaul Blakey parm = nla_data(tb[TCA_CT_PARMS]);
13297be8ef2cSDmytro Linkin index = parm->index;
13307be8ef2cSDmytro Linkin err = tcf_idr_check_alloc(tn, &index, a, bind);
1331b57dc7c1SPaul Blakey if (err < 0)
1332b57dc7c1SPaul Blakey return err;
1333b57dc7c1SPaul Blakey
1334b57dc7c1SPaul Blakey if (!err) {
1335e3822678SVlad Buslov err = tcf_idr_create_from_flags(tn, index, est, a,
1336e3822678SVlad Buslov &act_ct_ops, bind, flags);
1337b57dc7c1SPaul Blakey if (err) {
13387be8ef2cSDmytro Linkin tcf_idr_cleanup(tn, index);
1339b57dc7c1SPaul Blakey return err;
1340b57dc7c1SPaul Blakey }
1341b57dc7c1SPaul Blakey res = ACT_P_CREATED;
1342b57dc7c1SPaul Blakey } else {
1343b57dc7c1SPaul Blakey if (bind)
1344b57dc7c1SPaul Blakey return 0;
1345b57dc7c1SPaul Blakey
1346695176bfSCong Wang if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
1347b57dc7c1SPaul Blakey tcf_idr_release(*a, bind);
1348b57dc7c1SPaul Blakey return -EEXIST;
1349b57dc7c1SPaul Blakey }
1350b57dc7c1SPaul Blakey }
1351b57dc7c1SPaul Blakey err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
1352b57dc7c1SPaul Blakey if (err < 0)
1353b57dc7c1SPaul Blakey goto cleanup;
1354b57dc7c1SPaul Blakey
1355b57dc7c1SPaul Blakey c = to_ct(*a);
1356b57dc7c1SPaul Blakey
1357b57dc7c1SPaul Blakey params = kzalloc(sizeof(*params), GFP_KERNEL);
1358b57dc7c1SPaul Blakey if (unlikely(!params)) {
1359b57dc7c1SPaul Blakey err = -ENOMEM;
1360b57dc7c1SPaul Blakey goto cleanup;
1361b57dc7c1SPaul Blakey }
1362b57dc7c1SPaul Blakey
1363b57dc7c1SPaul Blakey err = tcf_ct_fill_params(net, params, parm, tb, extack);
1364b57dc7c1SPaul Blakey if (err)
1365b57dc7c1SPaul Blakey goto cleanup;
1366b57dc7c1SPaul Blakey
1367fc54d906SVlad Buslov err = tcf_ct_flow_table_get(net, params);
1368c34b961aSPaul Blakey if (err)
136919138941SXin Long goto cleanup;
1370c34b961aSPaul Blakey
1371b57dc7c1SPaul Blakey spin_lock_bh(&c->tcf_lock);
1372b57dc7c1SPaul Blakey goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
1373445d3749SPaul E. McKenney params = rcu_replace_pointer(c->params, params,
1374445d3749SPaul E. McKenney lockdep_is_held(&c->tcf_lock));
1375b57dc7c1SPaul Blakey spin_unlock_bh(&c->tcf_lock);
1376b57dc7c1SPaul Blakey
1377b57dc7c1SPaul Blakey if (goto_ch)
1378b57dc7c1SPaul Blakey tcf_chain_put_by_act(goto_ch);
1379b57dc7c1SPaul Blakey if (params)
138019138941SXin Long call_rcu(¶ms->rcu, tcf_ct_params_free_rcu);
1381b57dc7c1SPaul Blakey
1382b57dc7c1SPaul Blakey return res;
1383b57dc7c1SPaul Blakey
1384b57dc7c1SPaul Blakey cleanup:
1385b57dc7c1SPaul Blakey if (goto_ch)
1386b57dc7c1SPaul Blakey tcf_chain_put_by_act(goto_ch);
138719138941SXin Long if (params)
138819138941SXin Long tcf_ct_params_free(params);
1389b57dc7c1SPaul Blakey tcf_idr_release(*a, bind);
1390b57dc7c1SPaul Blakey return err;
1391b57dc7c1SPaul Blakey }
1392b57dc7c1SPaul Blakey
tcf_ct_cleanup(struct tc_action * a)1393b57dc7c1SPaul Blakey static void tcf_ct_cleanup(struct tc_action *a)
1394b57dc7c1SPaul Blakey {
1395b57dc7c1SPaul Blakey struct tcf_ct_params *params;
1396b57dc7c1SPaul Blakey struct tcf_ct *c = to_ct(a);
1397b57dc7c1SPaul Blakey
1398b57dc7c1SPaul Blakey params = rcu_dereference_protected(c->params, 1);
1399b57dc7c1SPaul Blakey if (params)
140019138941SXin Long call_rcu(¶ms->rcu, tcf_ct_params_free_rcu);
1401b57dc7c1SPaul Blakey }
1402b57dc7c1SPaul Blakey
tcf_ct_dump_key_val(struct sk_buff * skb,void * val,int val_type,void * mask,int mask_type,int len)1403b57dc7c1SPaul Blakey static int tcf_ct_dump_key_val(struct sk_buff *skb,
1404b57dc7c1SPaul Blakey void *val, int val_type,
1405b57dc7c1SPaul Blakey void *mask, int mask_type,
1406b57dc7c1SPaul Blakey int len)
1407b57dc7c1SPaul Blakey {
1408b57dc7c1SPaul Blakey int err;
1409b57dc7c1SPaul Blakey
1410b57dc7c1SPaul Blakey if (mask && !memchr_inv(mask, 0, len))
1411b57dc7c1SPaul Blakey return 0;
1412b57dc7c1SPaul Blakey
1413b57dc7c1SPaul Blakey err = nla_put(skb, val_type, len, val);
1414b57dc7c1SPaul Blakey if (err)
1415b57dc7c1SPaul Blakey return err;
1416b57dc7c1SPaul Blakey
1417b57dc7c1SPaul Blakey if (mask_type != TCA_CT_UNSPEC) {
1418b57dc7c1SPaul Blakey err = nla_put(skb, mask_type, len, mask);
1419b57dc7c1SPaul Blakey if (err)
1420b57dc7c1SPaul Blakey return err;
1421b57dc7c1SPaul Blakey }
1422b57dc7c1SPaul Blakey
1423b57dc7c1SPaul Blakey return 0;
1424b57dc7c1SPaul Blakey }
1425b57dc7c1SPaul Blakey
tcf_ct_dump_nat(struct sk_buff * skb,struct tcf_ct_params * p)1426b57dc7c1SPaul Blakey static int tcf_ct_dump_nat(struct sk_buff *skb, struct tcf_ct_params *p)
1427b57dc7c1SPaul Blakey {
1428b57dc7c1SPaul Blakey struct nf_nat_range2 *range = &p->range;
1429b57dc7c1SPaul Blakey
1430b57dc7c1SPaul Blakey if (!(p->ct_action & TCA_CT_ACT_NAT))
1431b57dc7c1SPaul Blakey return 0;
1432b57dc7c1SPaul Blakey
1433b57dc7c1SPaul Blakey if (!(p->ct_action & (TCA_CT_ACT_NAT_SRC | TCA_CT_ACT_NAT_DST)))
1434b57dc7c1SPaul Blakey return 0;
1435b57dc7c1SPaul Blakey
1436b57dc7c1SPaul Blakey if (range->flags & NF_NAT_RANGE_MAP_IPS) {
1437b57dc7c1SPaul Blakey if (p->ipv4_range) {
1438b57dc7c1SPaul Blakey if (nla_put_in_addr(skb, TCA_CT_NAT_IPV4_MIN,
1439b57dc7c1SPaul Blakey range->min_addr.ip))
1440b57dc7c1SPaul Blakey return -1;
1441b57dc7c1SPaul Blakey if (nla_put_in_addr(skb, TCA_CT_NAT_IPV4_MAX,
1442b57dc7c1SPaul Blakey range->max_addr.ip))
1443b57dc7c1SPaul Blakey return -1;
1444b57dc7c1SPaul Blakey } else {
1445b57dc7c1SPaul Blakey if (nla_put_in6_addr(skb, TCA_CT_NAT_IPV6_MIN,
1446b57dc7c1SPaul Blakey &range->min_addr.in6))
1447b57dc7c1SPaul Blakey return -1;
1448b57dc7c1SPaul Blakey if (nla_put_in6_addr(skb, TCA_CT_NAT_IPV6_MAX,
1449b57dc7c1SPaul Blakey &range->max_addr.in6))
1450b57dc7c1SPaul Blakey return -1;
1451b57dc7c1SPaul Blakey }
1452b57dc7c1SPaul Blakey }
1453b57dc7c1SPaul Blakey
1454b57dc7c1SPaul Blakey if (range->flags & NF_NAT_RANGE_PROTO_SPECIFIED) {
1455b57dc7c1SPaul Blakey if (nla_put_be16(skb, TCA_CT_NAT_PORT_MIN,
1456b57dc7c1SPaul Blakey range->min_proto.all))
1457b57dc7c1SPaul Blakey return -1;
1458b57dc7c1SPaul Blakey if (nla_put_be16(skb, TCA_CT_NAT_PORT_MAX,
1459b57dc7c1SPaul Blakey range->max_proto.all))
1460b57dc7c1SPaul Blakey return -1;
1461b57dc7c1SPaul Blakey }
1462b57dc7c1SPaul Blakey
1463b57dc7c1SPaul Blakey return 0;
1464b57dc7c1SPaul Blakey }
1465b57dc7c1SPaul Blakey
tcf_ct_dump_helper(struct sk_buff * skb,struct nf_conntrack_helper * helper)1466a21b06e7SXin Long static int tcf_ct_dump_helper(struct sk_buff *skb, struct nf_conntrack_helper *helper)
1467a21b06e7SXin Long {
1468a21b06e7SXin Long if (!helper)
1469a21b06e7SXin Long return 0;
1470a21b06e7SXin Long
1471a21b06e7SXin Long if (nla_put_string(skb, TCA_CT_HELPER_NAME, helper->name) ||
1472a21b06e7SXin Long nla_put_u8(skb, TCA_CT_HELPER_FAMILY, helper->tuple.src.l3num) ||
1473a21b06e7SXin Long nla_put_u8(skb, TCA_CT_HELPER_PROTO, helper->tuple.dst.protonum))
1474a21b06e7SXin Long return -1;
1475a21b06e7SXin Long
1476a21b06e7SXin Long return 0;
1477a21b06e7SXin Long }
1478a21b06e7SXin Long
tcf_ct_dump(struct sk_buff * skb,struct tc_action * a,int bind,int ref)1479b57dc7c1SPaul Blakey static inline int tcf_ct_dump(struct sk_buff *skb, struct tc_action *a,
1480b57dc7c1SPaul Blakey int bind, int ref)
1481b57dc7c1SPaul Blakey {
1482b57dc7c1SPaul Blakey unsigned char *b = skb_tail_pointer(skb);
1483b57dc7c1SPaul Blakey struct tcf_ct *c = to_ct(a);
1484b57dc7c1SPaul Blakey struct tcf_ct_params *p;
1485b57dc7c1SPaul Blakey
1486b57dc7c1SPaul Blakey struct tc_ct opt = {
1487b57dc7c1SPaul Blakey .index = c->tcf_index,
1488b57dc7c1SPaul Blakey .refcnt = refcount_read(&c->tcf_refcnt) - ref,
1489b57dc7c1SPaul Blakey .bindcnt = atomic_read(&c->tcf_bindcnt) - bind,
1490b57dc7c1SPaul Blakey };
1491b57dc7c1SPaul Blakey struct tcf_t t;
1492b57dc7c1SPaul Blakey
1493b57dc7c1SPaul Blakey spin_lock_bh(&c->tcf_lock);
1494b57dc7c1SPaul Blakey p = rcu_dereference_protected(c->params,
1495b57dc7c1SPaul Blakey lockdep_is_held(&c->tcf_lock));
1496b57dc7c1SPaul Blakey opt.action = c->tcf_action;
1497b57dc7c1SPaul Blakey
1498b57dc7c1SPaul Blakey if (tcf_ct_dump_key_val(skb,
1499b57dc7c1SPaul Blakey &p->ct_action, TCA_CT_ACTION,
1500b57dc7c1SPaul Blakey NULL, TCA_CT_UNSPEC,
1501b57dc7c1SPaul Blakey sizeof(p->ct_action)))
1502b57dc7c1SPaul Blakey goto nla_put_failure;
1503b57dc7c1SPaul Blakey
1504b57dc7c1SPaul Blakey if (p->ct_action & TCA_CT_ACT_CLEAR)
1505b57dc7c1SPaul Blakey goto skip_dump;
1506b57dc7c1SPaul Blakey
1507b57dc7c1SPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
1508b57dc7c1SPaul Blakey tcf_ct_dump_key_val(skb,
1509b57dc7c1SPaul Blakey &p->mark, TCA_CT_MARK,
1510b57dc7c1SPaul Blakey &p->mark_mask, TCA_CT_MARK_MASK,
1511b57dc7c1SPaul Blakey sizeof(p->mark)))
1512b57dc7c1SPaul Blakey goto nla_put_failure;
1513b57dc7c1SPaul Blakey
1514b57dc7c1SPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
1515b57dc7c1SPaul Blakey tcf_ct_dump_key_val(skb,
1516b57dc7c1SPaul Blakey p->labels, TCA_CT_LABELS,
1517b57dc7c1SPaul Blakey p->labels_mask, TCA_CT_LABELS_MASK,
1518b57dc7c1SPaul Blakey sizeof(p->labels)))
1519b57dc7c1SPaul Blakey goto nla_put_failure;
1520b57dc7c1SPaul Blakey
1521b57dc7c1SPaul Blakey if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
1522b57dc7c1SPaul Blakey tcf_ct_dump_key_val(skb,
1523b57dc7c1SPaul Blakey &p->zone, TCA_CT_ZONE,
1524b57dc7c1SPaul Blakey NULL, TCA_CT_UNSPEC,
1525b57dc7c1SPaul Blakey sizeof(p->zone)))
1526b57dc7c1SPaul Blakey goto nla_put_failure;
1527b57dc7c1SPaul Blakey
1528b57dc7c1SPaul Blakey if (tcf_ct_dump_nat(skb, p))
1529b57dc7c1SPaul Blakey goto nla_put_failure;
1530b57dc7c1SPaul Blakey
1531a21b06e7SXin Long if (tcf_ct_dump_helper(skb, p->helper))
1532a21b06e7SXin Long goto nla_put_failure;
1533a21b06e7SXin Long
1534b57dc7c1SPaul Blakey skip_dump:
1535b57dc7c1SPaul Blakey if (nla_put(skb, TCA_CT_PARMS, sizeof(opt), &opt))
1536b57dc7c1SPaul Blakey goto nla_put_failure;
1537b57dc7c1SPaul Blakey
1538b57dc7c1SPaul Blakey tcf_tm_dump(&t, &c->tcf_tm);
1539b57dc7c1SPaul Blakey if (nla_put_64bit(skb, TCA_CT_TM, sizeof(t), &t, TCA_CT_PAD))
1540b57dc7c1SPaul Blakey goto nla_put_failure;
1541b57dc7c1SPaul Blakey spin_unlock_bh(&c->tcf_lock);
1542b57dc7c1SPaul Blakey
1543b57dc7c1SPaul Blakey return skb->len;
1544b57dc7c1SPaul Blakey nla_put_failure:
1545b57dc7c1SPaul Blakey spin_unlock_bh(&c->tcf_lock);
1546b57dc7c1SPaul Blakey nlmsg_trim(skb, b);
1547b57dc7c1SPaul Blakey return -1;
1548b57dc7c1SPaul Blakey }
1549b57dc7c1SPaul Blakey
tcf_stats_update(struct tc_action * a,u64 bytes,u64 packets,u64 drops,u64 lastuse,bool hw)15504b61d3e8SPo Liu static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
15514b61d3e8SPo Liu u64 drops, u64 lastuse, bool hw)
1552b57dc7c1SPaul Blakey {
1553b57dc7c1SPaul Blakey struct tcf_ct *c = to_ct(a);
1554b57dc7c1SPaul Blakey
15554b61d3e8SPo Liu tcf_action_update_stats(a, bytes, packets, drops, hw);
1556b57dc7c1SPaul Blakey c->tcf_tm.lastuse = max_t(u64, c->tcf_tm.lastuse, lastuse);
1557b57dc7c1SPaul Blakey }
1558b57dc7c1SPaul Blakey
tcf_ct_offload_act_setup(struct tc_action * act,void * entry_data,u32 * index_inc,bool bind,struct netlink_ext_ack * extack)1559c54e1d92SBaowen Zheng static int tcf_ct_offload_act_setup(struct tc_action *act, void *entry_data,
1560c2ccf84eSIdo Schimmel u32 *index_inc, bool bind,
1561c2ccf84eSIdo Schimmel struct netlink_ext_ack *extack)
1562c54e1d92SBaowen Zheng {
1563c54e1d92SBaowen Zheng if (bind) {
1564c54e1d92SBaowen Zheng struct flow_action_entry *entry = entry_data;
1565c54e1d92SBaowen Zheng
15660a7c9d1fSXin Long if (tcf_ct_helper(act))
15670a7c9d1fSXin Long return -EOPNOTSUPP;
15680a7c9d1fSXin Long
1569c54e1d92SBaowen Zheng entry->id = FLOW_ACTION_CT;
1570c54e1d92SBaowen Zheng entry->ct.action = tcf_ct_action(act);
1571c54e1d92SBaowen Zheng entry->ct.zone = tcf_ct_zone(act);
1572c54e1d92SBaowen Zheng entry->ct.flow_table = tcf_ct_ft(act);
1573c54e1d92SBaowen Zheng *index_inc = 1;
1574c54e1d92SBaowen Zheng } else {
15758cbfe939SBaowen Zheng struct flow_offload_action *fl_action = entry_data;
15768cbfe939SBaowen Zheng
15778cbfe939SBaowen Zheng fl_action->id = FLOW_ACTION_CT;
1578c54e1d92SBaowen Zheng }
1579c54e1d92SBaowen Zheng
1580c54e1d92SBaowen Zheng return 0;
1581c54e1d92SBaowen Zheng }
1582c54e1d92SBaowen Zheng
1583b57dc7c1SPaul Blakey static struct tc_action_ops act_ct_ops = {
1584b57dc7c1SPaul Blakey .kind = "ct",
1585b57dc7c1SPaul Blakey .id = TCA_ID_CT,
1586b57dc7c1SPaul Blakey .owner = THIS_MODULE,
1587b57dc7c1SPaul Blakey .act = tcf_ct_act,
1588b57dc7c1SPaul Blakey .dump = tcf_ct_dump,
1589b57dc7c1SPaul Blakey .init = tcf_ct_init,
1590b57dc7c1SPaul Blakey .cleanup = tcf_ct_cleanup,
1591b57dc7c1SPaul Blakey .stats_update = tcf_stats_update,
1592c54e1d92SBaowen Zheng .offload_act_setup = tcf_ct_offload_act_setup,
1593b57dc7c1SPaul Blakey .size = sizeof(struct tcf_ct),
1594b57dc7c1SPaul Blakey };
1595b57dc7c1SPaul Blakey
ct_init_net(struct net * net)1596b57dc7c1SPaul Blakey static __net_init int ct_init_net(struct net *net)
1597b57dc7c1SPaul Blakey {
1598c593642cSPankaj Bharadiya unsigned int n_bits = sizeof_field(struct tcf_ct_params, labels) * 8;
1599acd0a7abSZhengchao Shao struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
1600b57dc7c1SPaul Blakey
1601b57dc7c1SPaul Blakey if (nf_connlabels_get(net, n_bits - 1)) {
1602b57dc7c1SPaul Blakey tn->labels = false;
1603b57dc7c1SPaul Blakey pr_err("act_ct: Failed to set connlabels length");
1604b57dc7c1SPaul Blakey } else {
1605b57dc7c1SPaul Blakey tn->labels = true;
1606b57dc7c1SPaul Blakey }
1607b57dc7c1SPaul Blakey
1608981471bdSCong Wang return tc_action_net_init(net, &tn->tn, &act_ct_ops);
1609b57dc7c1SPaul Blakey }
1610b57dc7c1SPaul Blakey
ct_exit_net(struct list_head * net_list)1611b57dc7c1SPaul Blakey static void __net_exit ct_exit_net(struct list_head *net_list)
1612b57dc7c1SPaul Blakey {
1613b57dc7c1SPaul Blakey struct net *net;
1614b57dc7c1SPaul Blakey
1615b57dc7c1SPaul Blakey rtnl_lock();
1616b57dc7c1SPaul Blakey list_for_each_entry(net, net_list, exit_list) {
1617acd0a7abSZhengchao Shao struct tc_ct_action_net *tn = net_generic(net, act_ct_ops.net_id);
1618b57dc7c1SPaul Blakey
1619b57dc7c1SPaul Blakey if (tn->labels)
1620b57dc7c1SPaul Blakey nf_connlabels_put(net);
1621b57dc7c1SPaul Blakey }
1622b57dc7c1SPaul Blakey rtnl_unlock();
1623b57dc7c1SPaul Blakey
1624acd0a7abSZhengchao Shao tc_action_net_exit(net_list, act_ct_ops.net_id);
1625b57dc7c1SPaul Blakey }
1626b57dc7c1SPaul Blakey
1627b57dc7c1SPaul Blakey static struct pernet_operations ct_net_ops = {
1628b57dc7c1SPaul Blakey .init = ct_init_net,
1629b57dc7c1SPaul Blakey .exit_batch = ct_exit_net,
1630acd0a7abSZhengchao Shao .id = &act_ct_ops.net_id,
1631b57dc7c1SPaul Blakey .size = sizeof(struct tc_ct_action_net),
1632b57dc7c1SPaul Blakey };
1633b57dc7c1SPaul Blakey
ct_init_module(void)1634b57dc7c1SPaul Blakey static int __init ct_init_module(void)
1635b57dc7c1SPaul Blakey {
1636c34b961aSPaul Blakey int err;
1637c34b961aSPaul Blakey
1638c34b961aSPaul Blakey act_ct_wq = alloc_ordered_workqueue("act_ct_workqueue", 0);
1639c34b961aSPaul Blakey if (!act_ct_wq)
1640c34b961aSPaul Blakey return -ENOMEM;
1641c34b961aSPaul Blakey
1642c34b961aSPaul Blakey err = tcf_ct_flow_tables_init();
1643c34b961aSPaul Blakey if (err)
1644c34b961aSPaul Blakey goto err_tbl_init;
1645c34b961aSPaul Blakey
1646c34b961aSPaul Blakey err = tcf_register_action(&act_ct_ops, &ct_net_ops);
1647c34b961aSPaul Blakey if (err)
1648c34b961aSPaul Blakey goto err_register;
1649c34b961aSPaul Blakey
1650c129412fSwenxu static_branch_inc(&tcf_frag_xmit_count);
1651c129412fSwenxu
1652c34b961aSPaul Blakey return 0;
1653c34b961aSPaul Blakey
1654c34b961aSPaul Blakey err_register:
1655c34b961aSPaul Blakey tcf_ct_flow_tables_uninit();
16568c5c51f5Sliujian err_tbl_init:
16578c5c51f5Sliujian destroy_workqueue(act_ct_wq);
1658c34b961aSPaul Blakey return err;
1659b57dc7c1SPaul Blakey }
1660b57dc7c1SPaul Blakey
ct_cleanup_module(void)1661b57dc7c1SPaul Blakey static void __exit ct_cleanup_module(void)
1662b57dc7c1SPaul Blakey {
1663c129412fSwenxu static_branch_dec(&tcf_frag_xmit_count);
1664b57dc7c1SPaul Blakey tcf_unregister_action(&act_ct_ops, &ct_net_ops);
1665c34b961aSPaul Blakey tcf_ct_flow_tables_uninit();
1666c34b961aSPaul Blakey destroy_workqueue(act_ct_wq);
1667b57dc7c1SPaul Blakey }
1668b57dc7c1SPaul Blakey
1669b57dc7c1SPaul Blakey module_init(ct_init_module);
1670b57dc7c1SPaul Blakey module_exit(ct_cleanup_module);
1671b57dc7c1SPaul Blakey MODULE_AUTHOR("Paul Blakey <paulb@mellanox.com>");
1672b57dc7c1SPaul Blakey MODULE_AUTHOR("Yossi Kuperman <yossiku@mellanox.com>");
1673b57dc7c1SPaul Blakey MODULE_AUTHOR("Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>");
1674b57dc7c1SPaul Blakey MODULE_DESCRIPTION("Connection tracking action");
1675b57dc7c1SPaul Blakey MODULE_LICENSE("GPL v2");
1676