1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
296518518SPatrick McHardy /*
3ef1f7df9SPatrick McHardy * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
496518518SPatrick McHardy *
596518518SPatrick McHardy * Development of this code funded by Astaro AG (http://www.astaro.com/)
696518518SPatrick McHardy */
796518518SPatrick McHardy
896518518SPatrick McHardy #include <linux/kernel.h>
996518518SPatrick McHardy #include <linux/init.h>
1096518518SPatrick McHardy #include <linux/module.h>
1196518518SPatrick McHardy #include <linux/netlink.h>
1296518518SPatrick McHardy #include <linux/netfilter.h>
138819efc9SPablo Neira Ayuso #include <linux/if_arp.h>
1496518518SPatrick McHardy #include <linux/netfilter/nf_tables.h>
1596518518SPatrick McHardy #include <net/netfilter/nf_tables_core.h>
16c9626a2cSPablo Neira Ayuso #include <net/netfilter/nf_tables_offload.h>
1796518518SPatrick McHardy #include <net/netfilter/nf_tables.h>
1896518518SPatrick McHardy
1996518518SPatrick McHardy struct nft_cmp_expr {
2096518518SPatrick McHardy struct nft_data data;
214f16d25cSPablo Neira Ayuso u8 sreg;
2296518518SPatrick McHardy u8 len;
2396518518SPatrick McHardy enum nft_cmp_ops op:8;
2496518518SPatrick McHardy };
2596518518SPatrick McHardy
nft_cmp_eval(const struct nft_expr * expr,struct nft_regs * regs,const struct nft_pktinfo * pkt)2610870dd8SFlorian Westphal void nft_cmp_eval(const struct nft_expr *expr,
27a55e22e9SPatrick McHardy struct nft_regs *regs,
2896518518SPatrick McHardy const struct nft_pktinfo *pkt)
2996518518SPatrick McHardy {
3096518518SPatrick McHardy const struct nft_cmp_expr *priv = nft_expr_priv(expr);
3196518518SPatrick McHardy int d;
3296518518SPatrick McHardy
33e562d860SPatrick McHardy d = memcmp(®s->data[priv->sreg], &priv->data, priv->len);
3496518518SPatrick McHardy switch (priv->op) {
3596518518SPatrick McHardy case NFT_CMP_EQ:
3696518518SPatrick McHardy if (d != 0)
3796518518SPatrick McHardy goto mismatch;
3896518518SPatrick McHardy break;
3996518518SPatrick McHardy case NFT_CMP_NEQ:
4096518518SPatrick McHardy if (d == 0)
4196518518SPatrick McHardy goto mismatch;
4296518518SPatrick McHardy break;
4396518518SPatrick McHardy case NFT_CMP_LT:
4496518518SPatrick McHardy if (d == 0)
4596518518SPatrick McHardy goto mismatch;
46954d8297SGustavo A. R. Silva fallthrough;
4796518518SPatrick McHardy case NFT_CMP_LTE:
4896518518SPatrick McHardy if (d > 0)
4996518518SPatrick McHardy goto mismatch;
5096518518SPatrick McHardy break;
5196518518SPatrick McHardy case NFT_CMP_GT:
5296518518SPatrick McHardy if (d == 0)
5396518518SPatrick McHardy goto mismatch;
54954d8297SGustavo A. R. Silva fallthrough;
5596518518SPatrick McHardy case NFT_CMP_GTE:
5696518518SPatrick McHardy if (d < 0)
5796518518SPatrick McHardy goto mismatch;
5896518518SPatrick McHardy break;
5996518518SPatrick McHardy }
6096518518SPatrick McHardy return;
6196518518SPatrick McHardy
6296518518SPatrick McHardy mismatch:
63a55e22e9SPatrick McHardy regs->verdict.code = NFT_BREAK;
6496518518SPatrick McHardy }
6596518518SPatrick McHardy
6696518518SPatrick McHardy static const struct nla_policy nft_cmp_policy[NFTA_CMP_MAX + 1] = {
6796518518SPatrick McHardy [NFTA_CMP_SREG] = { .type = NLA_U32 },
6896518518SPatrick McHardy [NFTA_CMP_OP] = { .type = NLA_U32 },
6996518518SPatrick McHardy [NFTA_CMP_DATA] = { .type = NLA_NESTED },
7096518518SPatrick McHardy };
7196518518SPatrick McHardy
nft_cmp_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])7296518518SPatrick McHardy static int nft_cmp_init(const struct nft_ctx *ctx, const struct nft_expr *expr,
7396518518SPatrick McHardy const struct nlattr * const tb[])
7496518518SPatrick McHardy {
7596518518SPatrick McHardy struct nft_cmp_expr *priv = nft_expr_priv(expr);
76341b6941SPablo Neira Ayuso struct nft_data_desc desc = {
77341b6941SPablo Neira Ayuso .type = NFT_DATA_VALUE,
78341b6941SPablo Neira Ayuso .size = sizeof(priv->data),
79341b6941SPablo Neira Ayuso };
8096518518SPatrick McHardy int err;
8196518518SPatrick McHardy
82341b6941SPablo Neira Ayuso err = nft_data_init(NULL, &priv->data, &desc, tb[NFTA_CMP_DATA]);
83fa5950e4SFlorian Westphal if (err < 0)
84fa5950e4SFlorian Westphal return err;
8596518518SPatrick McHardy
864f16d25cSPablo Neira Ayuso err = nft_parse_register_load(tb[NFTA_CMP_SREG], &priv->sreg, desc.len);
87d07db988SPatrick McHardy if (err < 0)
88d07db988SPatrick McHardy return err;
89d07db988SPatrick McHardy
90d07db988SPatrick McHardy priv->op = ntohl(nla_get_be32(tb[NFTA_CMP_OP]));
9196518518SPatrick McHardy priv->len = desc.len;
9296518518SPatrick McHardy return 0;
9396518518SPatrick McHardy }
9496518518SPatrick McHardy
nft_cmp_dump(struct sk_buff * skb,const struct nft_expr * expr,bool reset)957d34aa3eSPhil Sutter static int nft_cmp_dump(struct sk_buff *skb,
967d34aa3eSPhil Sutter const struct nft_expr *expr, bool reset)
9796518518SPatrick McHardy {
9896518518SPatrick McHardy const struct nft_cmp_expr *priv = nft_expr_priv(expr);
9996518518SPatrick McHardy
100b1c96ed3SPatrick McHardy if (nft_dump_register(skb, NFTA_CMP_SREG, priv->sreg))
10196518518SPatrick McHardy goto nla_put_failure;
10296518518SPatrick McHardy if (nla_put_be32(skb, NFTA_CMP_OP, htonl(priv->op)))
10396518518SPatrick McHardy goto nla_put_failure;
10496518518SPatrick McHardy
10596518518SPatrick McHardy if (nft_data_dump(skb, NFTA_CMP_DATA, &priv->data,
10696518518SPatrick McHardy NFT_DATA_VALUE, priv->len) < 0)
10796518518SPatrick McHardy goto nla_put_failure;
10896518518SPatrick McHardy return 0;
10996518518SPatrick McHardy
11096518518SPatrick McHardy nla_put_failure:
11196518518SPatrick McHardy return -1;
11296518518SPatrick McHardy }
11396518518SPatrick McHardy
114ff4d90a8SPablo Neira Ayuso union nft_cmp_offload_data {
115ff4d90a8SPablo Neira Ayuso u16 val16;
116ff4d90a8SPablo Neira Ayuso u32 val32;
117ff4d90a8SPablo Neira Ayuso u64 val64;
118ff4d90a8SPablo Neira Ayuso };
119ff4d90a8SPablo Neira Ayuso
nft_payload_n2h(union nft_cmp_offload_data * data,const u8 * val,u32 len)120ff4d90a8SPablo Neira Ayuso static void nft_payload_n2h(union nft_cmp_offload_data *data,
121ff4d90a8SPablo Neira Ayuso const u8 *val, u32 len)
122ff4d90a8SPablo Neira Ayuso {
123ff4d90a8SPablo Neira Ayuso switch (len) {
124ff4d90a8SPablo Neira Ayuso case 2:
125ffb3d9a3SFlorian Westphal data->val16 = ntohs(*((__be16 *)val));
126ff4d90a8SPablo Neira Ayuso break;
127ff4d90a8SPablo Neira Ayuso case 4:
128ffb3d9a3SFlorian Westphal data->val32 = ntohl(*((__be32 *)val));
129ff4d90a8SPablo Neira Ayuso break;
130ff4d90a8SPablo Neira Ayuso case 8:
131ffb3d9a3SFlorian Westphal data->val64 = be64_to_cpu(*((__be64 *)val));
132ff4d90a8SPablo Neira Ayuso break;
133ff4d90a8SPablo Neira Ayuso default:
134ff4d90a8SPablo Neira Ayuso WARN_ON_ONCE(1);
135ff4d90a8SPablo Neira Ayuso break;
136ff4d90a8SPablo Neira Ayuso }
137ff4d90a8SPablo Neira Ayuso }
138ff4d90a8SPablo Neira Ayuso
__nft_cmp_offload(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_cmp_expr * priv)139c9626a2cSPablo Neira Ayuso static int __nft_cmp_offload(struct nft_offload_ctx *ctx,
140c9626a2cSPablo Neira Ayuso struct nft_flow_rule *flow,
141c9626a2cSPablo Neira Ayuso const struct nft_cmp_expr *priv)
142c9626a2cSPablo Neira Ayuso {
143c9626a2cSPablo Neira Ayuso struct nft_offload_reg *reg = &ctx->regs[priv->sreg];
144ff4d90a8SPablo Neira Ayuso union nft_cmp_offload_data _data, _datamask;
145c9626a2cSPablo Neira Ayuso u8 *mask = (u8 *)&flow->match.mask;
146c9626a2cSPablo Neira Ayuso u8 *key = (u8 *)&flow->match.key;
147ff4d90a8SPablo Neira Ayuso u8 *data, *datamask;
148c9626a2cSPablo Neira Ayuso
149a5d45bc0SPablo Neira Ayuso if (priv->op != NFT_CMP_EQ || priv->len > reg->len)
150c9626a2cSPablo Neira Ayuso return -EOPNOTSUPP;
151c9626a2cSPablo Neira Ayuso
152ff4d90a8SPablo Neira Ayuso if (reg->flags & NFT_OFFLOAD_F_NETWORK2HOST) {
153ff4d90a8SPablo Neira Ayuso nft_payload_n2h(&_data, (u8 *)&priv->data, reg->len);
154ff4d90a8SPablo Neira Ayuso nft_payload_n2h(&_datamask, (u8 *)®->mask, reg->len);
155ff4d90a8SPablo Neira Ayuso data = (u8 *)&_data;
156ff4d90a8SPablo Neira Ayuso datamask = (u8 *)&_datamask;
157ff4d90a8SPablo Neira Ayuso } else {
158ff4d90a8SPablo Neira Ayuso data = (u8 *)&priv->data;
159ff4d90a8SPablo Neira Ayuso datamask = (u8 *)®->mask;
160ff4d90a8SPablo Neira Ayuso }
161ff4d90a8SPablo Neira Ayuso
162ff4d90a8SPablo Neira Ayuso memcpy(key + reg->offset, data, reg->len);
163ff4d90a8SPablo Neira Ayuso memcpy(mask + reg->offset, datamask, reg->len);
164c9626a2cSPablo Neira Ayuso
165*2b3082c6SRatheesh Kannoth flow->match.dissector.used_keys |= BIT_ULL(reg->key);
166c9626a2cSPablo Neira Ayuso flow->match.dissector.offset[reg->key] = reg->base_offset;
167c9626a2cSPablo Neira Ayuso
1688819efc9SPablo Neira Ayuso if (reg->key == FLOW_DISSECTOR_KEY_META &&
1698819efc9SPablo Neira Ayuso reg->offset == offsetof(struct nft_flow_key, meta.ingress_iftype) &&
1708819efc9SPablo Neira Ayuso nft_reg_load16(priv->data.data) != ARPHRD_ETHER)
1718819efc9SPablo Neira Ayuso return -EOPNOTSUPP;
1728819efc9SPablo Neira Ayuso
173a5d45bc0SPablo Neira Ayuso nft_offload_update_dependency(ctx, &priv->data, reg->len);
174c9626a2cSPablo Neira Ayuso
175c9626a2cSPablo Neira Ayuso return 0;
176c9626a2cSPablo Neira Ayuso }
177c9626a2cSPablo Neira Ayuso
nft_cmp_offload(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_expr * expr)178c9626a2cSPablo Neira Ayuso static int nft_cmp_offload(struct nft_offload_ctx *ctx,
179c9626a2cSPablo Neira Ayuso struct nft_flow_rule *flow,
180c9626a2cSPablo Neira Ayuso const struct nft_expr *expr)
181c9626a2cSPablo Neira Ayuso {
182c9626a2cSPablo Neira Ayuso const struct nft_cmp_expr *priv = nft_expr_priv(expr);
183c9626a2cSPablo Neira Ayuso
184c9626a2cSPablo Neira Ayuso return __nft_cmp_offload(ctx, flow, priv);
185c9626a2cSPablo Neira Ayuso }
186c9626a2cSPablo Neira Ayuso
187ef1f7df9SPatrick McHardy static const struct nft_expr_ops nft_cmp_ops = {
188ef1f7df9SPatrick McHardy .type = &nft_cmp_type,
18996518518SPatrick McHardy .size = NFT_EXPR_SIZE(sizeof(struct nft_cmp_expr)),
19096518518SPatrick McHardy .eval = nft_cmp_eval,
19196518518SPatrick McHardy .init = nft_cmp_init,
19296518518SPatrick McHardy .dump = nft_cmp_dump,
193b2d30654SPablo Neira Ayuso .reduce = NFT_REDUCE_READONLY,
194c9626a2cSPablo Neira Ayuso .offload = nft_cmp_offload,
195ef1f7df9SPatrick McHardy };
196ef1f7df9SPatrick McHardy
1976b772053SFlorian Westphal /* Calculate the mask for the nft_cmp_fast expression. On big endian the
1986b772053SFlorian Westphal * mask needs to include the *upper* bytes when interpreting that data as
1996b772053SFlorian Westphal * something smaller than the full u32, therefore a cpu_to_le32 is done.
2006b772053SFlorian Westphal */
nft_cmp_fast_mask(unsigned int len)2016b772053SFlorian Westphal static u32 nft_cmp_fast_mask(unsigned int len)
2026b772053SFlorian Westphal {
2036b772053SFlorian Westphal __le32 mask = cpu_to_le32(~0U >> (sizeof_field(struct nft_cmp_fast_expr,
2046b772053SFlorian Westphal data) * BITS_PER_BYTE - len));
2056b772053SFlorian Westphal
2066b772053SFlorian Westphal return (__force u32)mask;
2076b772053SFlorian Westphal }
2086b772053SFlorian Westphal
nft_cmp_fast_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])209cb7dbfd0SPatrick McHardy static int nft_cmp_fast_init(const struct nft_ctx *ctx,
210cb7dbfd0SPatrick McHardy const struct nft_expr *expr,
211cb7dbfd0SPatrick McHardy const struct nlattr * const tb[])
212cb7dbfd0SPatrick McHardy {
213cb7dbfd0SPatrick McHardy struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
214cb7dbfd0SPatrick McHardy struct nft_data data;
215341b6941SPablo Neira Ayuso struct nft_data_desc desc = {
216341b6941SPablo Neira Ayuso .type = NFT_DATA_VALUE,
217341b6941SPablo Neira Ayuso .size = sizeof(data),
218341b6941SPablo Neira Ayuso };
219cb7dbfd0SPatrick McHardy int err;
220cb7dbfd0SPatrick McHardy
221341b6941SPablo Neira Ayuso err = nft_data_init(NULL, &data, &desc, tb[NFTA_CMP_DATA]);
222fa5950e4SFlorian Westphal if (err < 0)
223fa5950e4SFlorian Westphal return err;
224cb7dbfd0SPatrick McHardy
2254f16d25cSPablo Neira Ayuso err = nft_parse_register_load(tb[NFTA_CMP_SREG], &priv->sreg, desc.len);
226d07db988SPatrick McHardy if (err < 0)
227d07db988SPatrick McHardy return err;
228d07db988SPatrick McHardy
229d07db988SPatrick McHardy desc.len *= BITS_PER_BYTE;
230d07db988SPatrick McHardy
2315f48846dSPhil Sutter priv->mask = nft_cmp_fast_mask(desc.len);
2325f48846dSPhil Sutter priv->data = data.data[0] & priv->mask;
233cb7dbfd0SPatrick McHardy priv->len = desc.len;
2345f48846dSPhil Sutter priv->inv = ntohl(nla_get_be32(tb[NFTA_CMP_OP])) != NFT_CMP_EQ;
235cb7dbfd0SPatrick McHardy return 0;
236cb7dbfd0SPatrick McHardy }
237cb7dbfd0SPatrick McHardy
nft_cmp_fast_offload(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_expr * expr)238c9626a2cSPablo Neira Ayuso static int nft_cmp_fast_offload(struct nft_offload_ctx *ctx,
239c9626a2cSPablo Neira Ayuso struct nft_flow_rule *flow,
240c9626a2cSPablo Neira Ayuso const struct nft_expr *expr)
241c9626a2cSPablo Neira Ayuso {
242c9626a2cSPablo Neira Ayuso const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
243c9626a2cSPablo Neira Ayuso struct nft_cmp_expr cmp = {
244c9626a2cSPablo Neira Ayuso .data = {
245c9626a2cSPablo Neira Ayuso .data = {
246c9626a2cSPablo Neira Ayuso [0] = priv->data,
247c9626a2cSPablo Neira Ayuso },
248c9626a2cSPablo Neira Ayuso },
249c9626a2cSPablo Neira Ayuso .sreg = priv->sreg,
250c9626a2cSPablo Neira Ayuso .len = priv->len / BITS_PER_BYTE,
2515f48846dSPhil Sutter .op = priv->inv ? NFT_CMP_NEQ : NFT_CMP_EQ,
252c9626a2cSPablo Neira Ayuso };
253c9626a2cSPablo Neira Ayuso
254c9626a2cSPablo Neira Ayuso return __nft_cmp_offload(ctx, flow, &cmp);
255c9626a2cSPablo Neira Ayuso }
256c9626a2cSPablo Neira Ayuso
nft_cmp_fast_dump(struct sk_buff * skb,const struct nft_expr * expr,bool reset)2577d34aa3eSPhil Sutter static int nft_cmp_fast_dump(struct sk_buff *skb,
2587d34aa3eSPhil Sutter const struct nft_expr *expr, bool reset)
259cb7dbfd0SPatrick McHardy {
260cb7dbfd0SPatrick McHardy const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
2615f48846dSPhil Sutter enum nft_cmp_ops op = priv->inv ? NFT_CMP_NEQ : NFT_CMP_EQ;
262cb7dbfd0SPatrick McHardy struct nft_data data;
263cb7dbfd0SPatrick McHardy
264b1c96ed3SPatrick McHardy if (nft_dump_register(skb, NFTA_CMP_SREG, priv->sreg))
265cb7dbfd0SPatrick McHardy goto nla_put_failure;
2665f48846dSPhil Sutter if (nla_put_be32(skb, NFTA_CMP_OP, htonl(op)))
267cb7dbfd0SPatrick McHardy goto nla_put_failure;
268cb7dbfd0SPatrick McHardy
269cb7dbfd0SPatrick McHardy data.data[0] = priv->data;
270cb7dbfd0SPatrick McHardy if (nft_data_dump(skb, NFTA_CMP_DATA, &data,
271cb7dbfd0SPatrick McHardy NFT_DATA_VALUE, priv->len / BITS_PER_BYTE) < 0)
272cb7dbfd0SPatrick McHardy goto nla_put_failure;
273cb7dbfd0SPatrick McHardy return 0;
274cb7dbfd0SPatrick McHardy
275cb7dbfd0SPatrick McHardy nla_put_failure:
276cb7dbfd0SPatrick McHardy return -1;
277cb7dbfd0SPatrick McHardy }
278cb7dbfd0SPatrick McHardy
279cb7dbfd0SPatrick McHardy const struct nft_expr_ops nft_cmp_fast_ops = {
280cb7dbfd0SPatrick McHardy .type = &nft_cmp_type,
281cb7dbfd0SPatrick McHardy .size = NFT_EXPR_SIZE(sizeof(struct nft_cmp_fast_expr)),
282cb7dbfd0SPatrick McHardy .eval = NULL, /* inlined */
283cb7dbfd0SPatrick McHardy .init = nft_cmp_fast_init,
284cb7dbfd0SPatrick McHardy .dump = nft_cmp_fast_dump,
285b2d30654SPablo Neira Ayuso .reduce = NFT_REDUCE_READONLY,
286c9626a2cSPablo Neira Ayuso .offload = nft_cmp_fast_offload,
287cb7dbfd0SPatrick McHardy };
288cb7dbfd0SPatrick McHardy
nft_cmp_mask(u32 bitlen)28923f68d46SPablo Neira Ayuso static u32 nft_cmp_mask(u32 bitlen)
29023f68d46SPablo Neira Ayuso {
29123f68d46SPablo Neira Ayuso return (__force u32)cpu_to_le32(~0U >> (sizeof(u32) * BITS_PER_BYTE - bitlen));
29223f68d46SPablo Neira Ayuso }
29323f68d46SPablo Neira Ayuso
nft_cmp16_fast_mask(struct nft_data * data,unsigned int bitlen)29423f68d46SPablo Neira Ayuso static void nft_cmp16_fast_mask(struct nft_data *data, unsigned int bitlen)
29523f68d46SPablo Neira Ayuso {
29623f68d46SPablo Neira Ayuso int len = bitlen / BITS_PER_BYTE;
29723f68d46SPablo Neira Ayuso int i, words = len / sizeof(u32);
29823f68d46SPablo Neira Ayuso
29923f68d46SPablo Neira Ayuso for (i = 0; i < words; i++) {
30023f68d46SPablo Neira Ayuso data->data[i] = 0xffffffff;
30123f68d46SPablo Neira Ayuso bitlen -= sizeof(u32) * BITS_PER_BYTE;
30223f68d46SPablo Neira Ayuso }
30323f68d46SPablo Neira Ayuso
30423f68d46SPablo Neira Ayuso if (len % sizeof(u32))
30523f68d46SPablo Neira Ayuso data->data[i++] = nft_cmp_mask(bitlen);
30623f68d46SPablo Neira Ayuso
30723f68d46SPablo Neira Ayuso for (; i < 4; i++)
30823f68d46SPablo Neira Ayuso data->data[i] = 0;
30923f68d46SPablo Neira Ayuso }
31023f68d46SPablo Neira Ayuso
nft_cmp16_fast_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])31123f68d46SPablo Neira Ayuso static int nft_cmp16_fast_init(const struct nft_ctx *ctx,
31223f68d46SPablo Neira Ayuso const struct nft_expr *expr,
31323f68d46SPablo Neira Ayuso const struct nlattr * const tb[])
31423f68d46SPablo Neira Ayuso {
31523f68d46SPablo Neira Ayuso struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr);
316341b6941SPablo Neira Ayuso struct nft_data_desc desc = {
317341b6941SPablo Neira Ayuso .type = NFT_DATA_VALUE,
318341b6941SPablo Neira Ayuso .size = sizeof(priv->data),
319341b6941SPablo Neira Ayuso };
32023f68d46SPablo Neira Ayuso int err;
32123f68d46SPablo Neira Ayuso
322341b6941SPablo Neira Ayuso err = nft_data_init(NULL, &priv->data, &desc, tb[NFTA_CMP_DATA]);
32323f68d46SPablo Neira Ayuso if (err < 0)
32423f68d46SPablo Neira Ayuso return err;
32523f68d46SPablo Neira Ayuso
32623f68d46SPablo Neira Ayuso err = nft_parse_register_load(tb[NFTA_CMP_SREG], &priv->sreg, desc.len);
32723f68d46SPablo Neira Ayuso if (err < 0)
32823f68d46SPablo Neira Ayuso return err;
32923f68d46SPablo Neira Ayuso
33023f68d46SPablo Neira Ayuso nft_cmp16_fast_mask(&priv->mask, desc.len * BITS_PER_BYTE);
33123f68d46SPablo Neira Ayuso priv->inv = ntohl(nla_get_be32(tb[NFTA_CMP_OP])) != NFT_CMP_EQ;
33223f68d46SPablo Neira Ayuso priv->len = desc.len;
33323f68d46SPablo Neira Ayuso
33423f68d46SPablo Neira Ayuso return 0;
33523f68d46SPablo Neira Ayuso }
33623f68d46SPablo Neira Ayuso
nft_cmp16_fast_offload(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_expr * expr)33723f68d46SPablo Neira Ayuso static int nft_cmp16_fast_offload(struct nft_offload_ctx *ctx,
33823f68d46SPablo Neira Ayuso struct nft_flow_rule *flow,
33923f68d46SPablo Neira Ayuso const struct nft_expr *expr)
34023f68d46SPablo Neira Ayuso {
34123f68d46SPablo Neira Ayuso const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr);
34223f68d46SPablo Neira Ayuso struct nft_cmp_expr cmp = {
34323f68d46SPablo Neira Ayuso .data = priv->data,
34423f68d46SPablo Neira Ayuso .sreg = priv->sreg,
34523f68d46SPablo Neira Ayuso .len = priv->len,
34623f68d46SPablo Neira Ayuso .op = priv->inv ? NFT_CMP_NEQ : NFT_CMP_EQ,
34723f68d46SPablo Neira Ayuso };
34823f68d46SPablo Neira Ayuso
34923f68d46SPablo Neira Ayuso return __nft_cmp_offload(ctx, flow, &cmp);
35023f68d46SPablo Neira Ayuso }
35123f68d46SPablo Neira Ayuso
nft_cmp16_fast_dump(struct sk_buff * skb,const struct nft_expr * expr,bool reset)3527d34aa3eSPhil Sutter static int nft_cmp16_fast_dump(struct sk_buff *skb,
3537d34aa3eSPhil Sutter const struct nft_expr *expr, bool reset)
35423f68d46SPablo Neira Ayuso {
35523f68d46SPablo Neira Ayuso const struct nft_cmp16_fast_expr *priv = nft_expr_priv(expr);
35623f68d46SPablo Neira Ayuso enum nft_cmp_ops op = priv->inv ? NFT_CMP_NEQ : NFT_CMP_EQ;
35723f68d46SPablo Neira Ayuso
35823f68d46SPablo Neira Ayuso if (nft_dump_register(skb, NFTA_CMP_SREG, priv->sreg))
35923f68d46SPablo Neira Ayuso goto nla_put_failure;
36023f68d46SPablo Neira Ayuso if (nla_put_be32(skb, NFTA_CMP_OP, htonl(op)))
36123f68d46SPablo Neira Ayuso goto nla_put_failure;
36223f68d46SPablo Neira Ayuso
36323f68d46SPablo Neira Ayuso if (nft_data_dump(skb, NFTA_CMP_DATA, &priv->data,
36423f68d46SPablo Neira Ayuso NFT_DATA_VALUE, priv->len) < 0)
36523f68d46SPablo Neira Ayuso goto nla_put_failure;
36623f68d46SPablo Neira Ayuso return 0;
36723f68d46SPablo Neira Ayuso
36823f68d46SPablo Neira Ayuso nla_put_failure:
36923f68d46SPablo Neira Ayuso return -1;
37023f68d46SPablo Neira Ayuso }
37123f68d46SPablo Neira Ayuso
37223f68d46SPablo Neira Ayuso
37323f68d46SPablo Neira Ayuso const struct nft_expr_ops nft_cmp16_fast_ops = {
37423f68d46SPablo Neira Ayuso .type = &nft_cmp_type,
37523f68d46SPablo Neira Ayuso .size = NFT_EXPR_SIZE(sizeof(struct nft_cmp16_fast_expr)),
37623f68d46SPablo Neira Ayuso .eval = NULL, /* inlined */
37723f68d46SPablo Neira Ayuso .init = nft_cmp16_fast_init,
37823f68d46SPablo Neira Ayuso .dump = nft_cmp16_fast_dump,
379b2d30654SPablo Neira Ayuso .reduce = NFT_REDUCE_READONLY,
38023f68d46SPablo Neira Ayuso .offload = nft_cmp16_fast_offload,
38123f68d46SPablo Neira Ayuso };
38223f68d46SPablo Neira Ayuso
3830ca743a5SPablo Neira Ayuso static const struct nft_expr_ops *
nft_cmp_select_ops(const struct nft_ctx * ctx,const struct nlattr * const tb[])3840ca743a5SPablo Neira Ayuso nft_cmp_select_ops(const struct nft_ctx *ctx, const struct nlattr * const tb[])
385cb7dbfd0SPatrick McHardy {
386cb7dbfd0SPatrick McHardy struct nft_data data;
387341b6941SPablo Neira Ayuso struct nft_data_desc desc = {
388341b6941SPablo Neira Ayuso .type = NFT_DATA_VALUE,
389341b6941SPablo Neira Ayuso .size = sizeof(data),
390341b6941SPablo Neira Ayuso };
391cb7dbfd0SPatrick McHardy enum nft_cmp_ops op;
39223f68d46SPablo Neira Ayuso u8 sreg;
393cb7dbfd0SPatrick McHardy int err;
394cb7dbfd0SPatrick McHardy
395cb7dbfd0SPatrick McHardy if (tb[NFTA_CMP_SREG] == NULL ||
396cb7dbfd0SPatrick McHardy tb[NFTA_CMP_OP] == NULL ||
397cb7dbfd0SPatrick McHardy tb[NFTA_CMP_DATA] == NULL)
398cb7dbfd0SPatrick McHardy return ERR_PTR(-EINVAL);
399cb7dbfd0SPatrick McHardy
400cb7dbfd0SPatrick McHardy op = ntohl(nla_get_be32(tb[NFTA_CMP_OP]));
401cb7dbfd0SPatrick McHardy switch (op) {
402cb7dbfd0SPatrick McHardy case NFT_CMP_EQ:
403cb7dbfd0SPatrick McHardy case NFT_CMP_NEQ:
404cb7dbfd0SPatrick McHardy case NFT_CMP_LT:
405cb7dbfd0SPatrick McHardy case NFT_CMP_LTE:
406cb7dbfd0SPatrick McHardy case NFT_CMP_GT:
407cb7dbfd0SPatrick McHardy case NFT_CMP_GTE:
408cb7dbfd0SPatrick McHardy break;
409cb7dbfd0SPatrick McHardy default:
410cb7dbfd0SPatrick McHardy return ERR_PTR(-EINVAL);
411cb7dbfd0SPatrick McHardy }
412cb7dbfd0SPatrick McHardy
413341b6941SPablo Neira Ayuso err = nft_data_init(NULL, &data, &desc, tb[NFTA_CMP_DATA]);
414cb7dbfd0SPatrick McHardy if (err < 0)
415cb7dbfd0SPatrick McHardy return ERR_PTR(err);
416cb7dbfd0SPatrick McHardy
41723f68d46SPablo Neira Ayuso sreg = ntohl(nla_get_be32(tb[NFTA_CMP_SREG]));
41871df14b0SPablo Neira Ayuso
41923f68d46SPablo Neira Ayuso if (op == NFT_CMP_EQ || op == NFT_CMP_NEQ) {
42023f68d46SPablo Neira Ayuso if (desc.len <= sizeof(u32))
42123f68d46SPablo Neira Ayuso return &nft_cmp_fast_ops;
42223f68d46SPablo Neira Ayuso else if (desc.len <= sizeof(data) &&
42323f68d46SPablo Neira Ayuso ((sreg >= NFT_REG_1 && sreg <= NFT_REG_4) ||
42423f68d46SPablo Neira Ayuso (sreg >= NFT_REG32_00 && sreg <= NFT_REG32_12 && sreg % 2 == 0)))
42523f68d46SPablo Neira Ayuso return &nft_cmp16_fast_ops;
42623f68d46SPablo Neira Ayuso }
427cb7dbfd0SPatrick McHardy return &nft_cmp_ops;
428cb7dbfd0SPatrick McHardy }
429cb7dbfd0SPatrick McHardy
4304e24877eSLiping Zhang struct nft_expr_type nft_cmp_type __read_mostly = {
431ef1f7df9SPatrick McHardy .name = "cmp",
432cb7dbfd0SPatrick McHardy .select_ops = nft_cmp_select_ops,
43396518518SPatrick McHardy .policy = nft_cmp_policy,
43496518518SPatrick McHardy .maxattr = NFTA_CMP_MAX,
435ef1f7df9SPatrick McHardy .owner = THIS_MODULE,
43696518518SPatrick McHardy };
437