1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include <linux/init.h> 3 #include <linux/module.h> 4 #include <linux/netfilter.h> 5 #include <net/flow_offload.h> 6 #include <net/netfilter/nf_tables.h> 7 #include <net/netfilter/nf_tables_offload.h> 8 #include <net/pkt_cls.h> 9 10 static struct nft_flow_rule *nft_flow_rule_alloc(int num_actions) 11 { 12 struct nft_flow_rule *flow; 13 14 flow = kzalloc(sizeof(struct nft_flow_rule), GFP_KERNEL); 15 if (!flow) 16 return NULL; 17 18 flow->rule = flow_rule_alloc(num_actions); 19 if (!flow->rule) { 20 kfree(flow); 21 return NULL; 22 } 23 24 flow->rule->match.dissector = &flow->match.dissector; 25 flow->rule->match.mask = &flow->match.mask; 26 flow->rule->match.key = &flow->match.key; 27 28 return flow; 29 } 30 31 void nft_flow_rule_set_addr_type(struct nft_flow_rule *flow, 32 enum flow_dissector_key_id addr_type) 33 { 34 struct nft_flow_match *match = &flow->match; 35 struct nft_flow_key *mask = &match->mask; 36 struct nft_flow_key *key = &match->key; 37 38 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) 39 return; 40 41 key->control.addr_type = addr_type; 42 mask->control.addr_type = 0xffff; 43 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CONTROL); 44 match->dissector.offset[FLOW_DISSECTOR_KEY_CONTROL] = 45 offsetof(struct nft_flow_key, control); 46 } 47 48 struct nft_offload_ethertype { 49 __be16 value; 50 __be16 mask; 51 }; 52 53 static void nft_flow_rule_transfer_vlan(struct nft_offload_ctx *ctx, 54 struct nft_flow_rule *flow) 55 { 56 struct nft_flow_match *match = &flow->match; 57 struct nft_offload_ethertype ethertype = { 58 .value = match->key.basic.n_proto, 59 .mask = match->mask.basic.n_proto, 60 }; 61 62 if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_VLAN) && 63 (match->key.vlan.vlan_tpid == htons(ETH_P_8021Q) || 64 match->key.vlan.vlan_tpid == htons(ETH_P_8021AD))) { 65 match->key.basic.n_proto = match->key.cvlan.vlan_tpid; 66 match->mask.basic.n_proto = match->mask.cvlan.vlan_tpid; 67 match->key.cvlan.vlan_tpid = match->key.vlan.vlan_tpid; 68 match->mask.cvlan.vlan_tpid = match->mask.vlan.vlan_tpid; 69 match->key.vlan.vlan_tpid = ethertype.value; 70 match->mask.vlan.vlan_tpid = ethertype.mask; 71 match->dissector.offset[FLOW_DISSECTOR_KEY_CVLAN] = 72 offsetof(struct nft_flow_key, cvlan); 73 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_CVLAN); 74 } else if (match->dissector.used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC) && 75 (match->key.basic.n_proto == htons(ETH_P_8021Q) || 76 match->key.basic.n_proto == htons(ETH_P_8021AD))) { 77 match->key.basic.n_proto = match->key.vlan.vlan_tpid; 78 match->mask.basic.n_proto = match->mask.vlan.vlan_tpid; 79 match->key.vlan.vlan_tpid = ethertype.value; 80 match->mask.vlan.vlan_tpid = ethertype.mask; 81 match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] = 82 offsetof(struct nft_flow_key, vlan); 83 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_VLAN); 84 } 85 } 86 87 struct nft_flow_rule *nft_flow_rule_create(struct net *net, 88 const struct nft_rule *rule) 89 { 90 struct nft_offload_ctx *ctx; 91 struct nft_flow_rule *flow; 92 int num_actions = 0, err; 93 struct nft_expr *expr; 94 95 expr = nft_expr_first(rule); 96 while (nft_expr_more(rule, expr)) { 97 if (expr->ops->offload_action && 98 expr->ops->offload_action(expr)) 99 num_actions++; 100 101 expr = nft_expr_next(expr); 102 } 103 104 if (num_actions == 0) 105 return ERR_PTR(-EOPNOTSUPP); 106 107 flow = nft_flow_rule_alloc(num_actions); 108 if (!flow) 109 return ERR_PTR(-ENOMEM); 110 111 expr = nft_expr_first(rule); 112 113 ctx = kzalloc(sizeof(struct nft_offload_ctx), GFP_KERNEL); 114 if (!ctx) { 115 err = -ENOMEM; 116 goto err_out; 117 } 118 ctx->net = net; 119 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; 120 121 while (nft_expr_more(rule, expr)) { 122 if (!expr->ops->offload) { 123 err = -EOPNOTSUPP; 124 goto err_out; 125 } 126 err = expr->ops->offload(ctx, flow, expr); 127 if (err < 0) 128 goto err_out; 129 130 expr = nft_expr_next(expr); 131 } 132 nft_flow_rule_transfer_vlan(ctx, flow); 133 134 flow->proto = ctx->dep.l3num; 135 kfree(ctx); 136 137 return flow; 138 err_out: 139 kfree(ctx); 140 nft_flow_rule_destroy(flow); 141 142 return ERR_PTR(err); 143 } 144 145 void nft_flow_rule_destroy(struct nft_flow_rule *flow) 146 { 147 struct flow_action_entry *entry; 148 int i; 149 150 flow_action_for_each(i, entry, &flow->rule->action) { 151 switch (entry->id) { 152 case FLOW_ACTION_REDIRECT: 153 case FLOW_ACTION_MIRRED: 154 dev_put(entry->dev); 155 break; 156 default: 157 break; 158 } 159 } 160 kfree(flow->rule); 161 kfree(flow); 162 } 163 164 void nft_offload_set_dependency(struct nft_offload_ctx *ctx, 165 enum nft_offload_dep_type type) 166 { 167 ctx->dep.type = type; 168 } 169 170 void nft_offload_update_dependency(struct nft_offload_ctx *ctx, 171 const void *data, u32 len) 172 { 173 switch (ctx->dep.type) { 174 case NFT_OFFLOAD_DEP_NETWORK: 175 WARN_ON(len != sizeof(__u16)); 176 memcpy(&ctx->dep.l3num, data, sizeof(__u16)); 177 break; 178 case NFT_OFFLOAD_DEP_TRANSPORT: 179 WARN_ON(len != sizeof(__u8)); 180 memcpy(&ctx->dep.protonum, data, sizeof(__u8)); 181 break; 182 default: 183 break; 184 } 185 ctx->dep.type = NFT_OFFLOAD_DEP_UNSPEC; 186 } 187 188 static void nft_flow_offload_common_init(struct flow_cls_common_offload *common, 189 __be16 proto, int priority, 190 struct netlink_ext_ack *extack) 191 { 192 common->protocol = proto; 193 common->prio = priority; 194 common->extack = extack; 195 } 196 197 static int nft_setup_cb_call(enum tc_setup_type type, void *type_data, 198 struct list_head *cb_list) 199 { 200 struct flow_block_cb *block_cb; 201 int err; 202 203 list_for_each_entry(block_cb, cb_list, list) { 204 err = block_cb->cb(type, type_data, block_cb->cb_priv); 205 if (err < 0) 206 return err; 207 } 208 return 0; 209 } 210 211 static int nft_chain_offload_priority(const struct nft_base_chain *basechain) 212 { 213 if (basechain->ops.priority <= 0 || 214 basechain->ops.priority > USHRT_MAX) 215 return -1; 216 217 return 0; 218 } 219 220 bool nft_chain_offload_support(const struct nft_base_chain *basechain) 221 { 222 struct net_device *dev; 223 struct nft_hook *hook; 224 225 if (nft_chain_offload_priority(basechain) < 0) 226 return false; 227 228 list_for_each_entry(hook, &basechain->hook_list, list) { 229 if (hook->ops.pf != NFPROTO_NETDEV || 230 hook->ops.hooknum != NF_NETDEV_INGRESS) 231 return false; 232 233 dev = hook->ops.dev; 234 if (!dev->netdev_ops->ndo_setup_tc && !flow_indr_dev_exists()) 235 return false; 236 } 237 238 return true; 239 } 240 241 static void nft_flow_cls_offload_setup(struct flow_cls_offload *cls_flow, 242 const struct nft_base_chain *basechain, 243 const struct nft_rule *rule, 244 const struct nft_flow_rule *flow, 245 struct netlink_ext_ack *extack, 246 enum flow_cls_command command) 247 { 248 __be16 proto = ETH_P_ALL; 249 250 memset(cls_flow, 0, sizeof(*cls_flow)); 251 252 if (flow) 253 proto = flow->proto; 254 255 nft_flow_offload_common_init(&cls_flow->common, proto, 256 basechain->ops.priority, extack); 257 cls_flow->command = command; 258 cls_flow->cookie = (unsigned long) rule; 259 if (flow) 260 cls_flow->rule = flow->rule; 261 } 262 263 static int nft_flow_offload_cmd(const struct nft_chain *chain, 264 const struct nft_rule *rule, 265 struct nft_flow_rule *flow, 266 enum flow_cls_command command, 267 struct flow_cls_offload *cls_flow) 268 { 269 struct netlink_ext_ack extack = {}; 270 struct nft_base_chain *basechain; 271 272 if (!nft_is_base_chain(chain)) 273 return -EOPNOTSUPP; 274 275 basechain = nft_base_chain(chain); 276 nft_flow_cls_offload_setup(cls_flow, basechain, rule, flow, &extack, 277 command); 278 279 return nft_setup_cb_call(TC_SETUP_CLSFLOWER, cls_flow, 280 &basechain->flow_block.cb_list); 281 } 282 283 static int nft_flow_offload_rule(const struct nft_chain *chain, 284 struct nft_rule *rule, 285 struct nft_flow_rule *flow, 286 enum flow_cls_command command) 287 { 288 struct flow_cls_offload cls_flow; 289 290 return nft_flow_offload_cmd(chain, rule, flow, command, &cls_flow); 291 } 292 293 int nft_flow_rule_stats(const struct nft_chain *chain, 294 const struct nft_rule *rule) 295 { 296 struct flow_cls_offload cls_flow = {}; 297 struct nft_expr *expr, *next; 298 int err; 299 300 err = nft_flow_offload_cmd(chain, rule, NULL, FLOW_CLS_STATS, 301 &cls_flow); 302 if (err < 0) 303 return err; 304 305 nft_rule_for_each_expr(expr, next, rule) { 306 if (expr->ops->offload_stats) 307 expr->ops->offload_stats(expr, &cls_flow.stats); 308 } 309 310 return 0; 311 } 312 313 static int nft_flow_offload_bind(struct flow_block_offload *bo, 314 struct nft_base_chain *basechain) 315 { 316 list_splice(&bo->cb_list, &basechain->flow_block.cb_list); 317 return 0; 318 } 319 320 static int nft_flow_offload_unbind(struct flow_block_offload *bo, 321 struct nft_base_chain *basechain) 322 { 323 struct flow_block_cb *block_cb, *next; 324 struct flow_cls_offload cls_flow; 325 struct netlink_ext_ack extack; 326 struct nft_chain *chain; 327 struct nft_rule *rule; 328 329 chain = &basechain->chain; 330 list_for_each_entry(rule, &chain->rules, list) { 331 memset(&extack, 0, sizeof(extack)); 332 nft_flow_cls_offload_setup(&cls_flow, basechain, rule, NULL, 333 &extack, FLOW_CLS_DESTROY); 334 nft_setup_cb_call(TC_SETUP_CLSFLOWER, &cls_flow, &bo->cb_list); 335 } 336 337 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 338 list_del(&block_cb->list); 339 flow_block_cb_free(block_cb); 340 } 341 342 return 0; 343 } 344 345 static int nft_block_setup(struct nft_base_chain *basechain, 346 struct flow_block_offload *bo, 347 enum flow_block_command cmd) 348 { 349 int err; 350 351 switch (cmd) { 352 case FLOW_BLOCK_BIND: 353 err = nft_flow_offload_bind(bo, basechain); 354 break; 355 case FLOW_BLOCK_UNBIND: 356 err = nft_flow_offload_unbind(bo, basechain); 357 break; 358 default: 359 WARN_ON_ONCE(1); 360 err = -EOPNOTSUPP; 361 } 362 363 return err; 364 } 365 366 static void nft_flow_block_offload_init(struct flow_block_offload *bo, 367 struct net *net, 368 enum flow_block_command cmd, 369 struct nft_base_chain *basechain, 370 struct netlink_ext_ack *extack) 371 { 372 memset(bo, 0, sizeof(*bo)); 373 bo->net = net; 374 bo->block = &basechain->flow_block; 375 bo->command = cmd; 376 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; 377 bo->extack = extack; 378 bo->cb_list_head = &basechain->flow_block.cb_list; 379 INIT_LIST_HEAD(&bo->cb_list); 380 } 381 382 static int nft_block_offload_cmd(struct nft_base_chain *chain, 383 struct net_device *dev, 384 enum flow_block_command cmd) 385 { 386 struct netlink_ext_ack extack = {}; 387 struct flow_block_offload bo; 388 int err; 389 390 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, chain, &extack); 391 392 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); 393 if (err < 0) 394 return err; 395 396 return nft_block_setup(chain, &bo, cmd); 397 } 398 399 static void nft_indr_block_cleanup(struct flow_block_cb *block_cb) 400 { 401 struct nft_base_chain *basechain = block_cb->indr.data; 402 struct net_device *dev = block_cb->indr.dev; 403 struct netlink_ext_ack extack = {}; 404 struct nftables_pernet *nft_net; 405 struct net *net = dev_net(dev); 406 struct flow_block_offload bo; 407 408 nft_flow_block_offload_init(&bo, dev_net(dev), FLOW_BLOCK_UNBIND, 409 basechain, &extack); 410 nft_net = nft_pernet(net); 411 mutex_lock(&nft_net->commit_mutex); 412 list_del(&block_cb->driver_list); 413 list_move(&block_cb->list, &bo.cb_list); 414 nft_flow_offload_unbind(&bo, basechain); 415 mutex_unlock(&nft_net->commit_mutex); 416 } 417 418 static int nft_indr_block_offload_cmd(struct nft_base_chain *basechain, 419 struct net_device *dev, 420 enum flow_block_command cmd) 421 { 422 struct netlink_ext_ack extack = {}; 423 struct flow_block_offload bo; 424 int err; 425 426 nft_flow_block_offload_init(&bo, dev_net(dev), cmd, basechain, &extack); 427 428 err = flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_BLOCK, basechain, &bo, 429 nft_indr_block_cleanup); 430 if (err < 0) 431 return err; 432 433 if (list_empty(&bo.cb_list)) 434 return -EOPNOTSUPP; 435 436 return nft_block_setup(basechain, &bo, cmd); 437 } 438 439 static int nft_chain_offload_cmd(struct nft_base_chain *basechain, 440 struct net_device *dev, 441 enum flow_block_command cmd) 442 { 443 int err; 444 445 if (dev->netdev_ops->ndo_setup_tc) 446 err = nft_block_offload_cmd(basechain, dev, cmd); 447 else 448 err = nft_indr_block_offload_cmd(basechain, dev, cmd); 449 450 return err; 451 } 452 453 static int nft_flow_block_chain(struct nft_base_chain *basechain, 454 const struct net_device *this_dev, 455 enum flow_block_command cmd) 456 { 457 struct net_device *dev; 458 struct nft_hook *hook; 459 int err, i = 0; 460 461 list_for_each_entry(hook, &basechain->hook_list, list) { 462 dev = hook->ops.dev; 463 if (this_dev && this_dev != dev) 464 continue; 465 466 err = nft_chain_offload_cmd(basechain, dev, cmd); 467 if (err < 0 && cmd == FLOW_BLOCK_BIND) { 468 if (!this_dev) 469 goto err_flow_block; 470 471 return err; 472 } 473 i++; 474 } 475 476 return 0; 477 478 err_flow_block: 479 list_for_each_entry(hook, &basechain->hook_list, list) { 480 if (i-- <= 0) 481 break; 482 483 dev = hook->ops.dev; 484 nft_chain_offload_cmd(basechain, dev, FLOW_BLOCK_UNBIND); 485 } 486 return err; 487 } 488 489 static int nft_flow_offload_chain(struct nft_chain *chain, u8 *ppolicy, 490 enum flow_block_command cmd) 491 { 492 struct nft_base_chain *basechain; 493 u8 policy; 494 495 if (!nft_is_base_chain(chain)) 496 return -EOPNOTSUPP; 497 498 basechain = nft_base_chain(chain); 499 policy = ppolicy ? *ppolicy : basechain->policy; 500 501 /* Only default policy to accept is supported for now. */ 502 if (cmd == FLOW_BLOCK_BIND && policy == NF_DROP) 503 return -EOPNOTSUPP; 504 505 return nft_flow_block_chain(basechain, NULL, cmd); 506 } 507 508 static void nft_flow_rule_offload_abort(struct net *net, 509 struct nft_trans *trans) 510 { 511 struct nftables_pernet *nft_net = nft_pernet(net); 512 int err = 0; 513 514 list_for_each_entry_continue_reverse(trans, &nft_net->commit_list, list) { 515 if (trans->ctx.family != NFPROTO_NETDEV) 516 continue; 517 518 switch (trans->msg_type) { 519 case NFT_MSG_NEWCHAIN: 520 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) || 521 nft_trans_chain_update(trans)) 522 continue; 523 524 err = nft_flow_offload_chain(trans->ctx.chain, NULL, 525 FLOW_BLOCK_UNBIND); 526 break; 527 case NFT_MSG_DELCHAIN: 528 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 529 continue; 530 531 err = nft_flow_offload_chain(trans->ctx.chain, NULL, 532 FLOW_BLOCK_BIND); 533 break; 534 case NFT_MSG_NEWRULE: 535 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 536 continue; 537 538 err = nft_flow_offload_rule(trans->ctx.chain, 539 nft_trans_rule(trans), 540 NULL, FLOW_CLS_DESTROY); 541 break; 542 case NFT_MSG_DELRULE: 543 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 544 continue; 545 546 err = nft_flow_offload_rule(trans->ctx.chain, 547 nft_trans_rule(trans), 548 nft_trans_flow_rule(trans), 549 FLOW_CLS_REPLACE); 550 break; 551 } 552 553 if (WARN_ON_ONCE(err)) 554 break; 555 } 556 } 557 558 int nft_flow_rule_offload_commit(struct net *net) 559 { 560 struct nftables_pernet *nft_net = nft_pernet(net); 561 struct nft_trans *trans; 562 int err = 0; 563 u8 policy; 564 565 list_for_each_entry(trans, &nft_net->commit_list, list) { 566 if (trans->ctx.family != NFPROTO_NETDEV) 567 continue; 568 569 switch (trans->msg_type) { 570 case NFT_MSG_NEWCHAIN: 571 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD) || 572 nft_trans_chain_update(trans)) 573 continue; 574 575 policy = nft_trans_chain_policy(trans); 576 err = nft_flow_offload_chain(trans->ctx.chain, &policy, 577 FLOW_BLOCK_BIND); 578 break; 579 case NFT_MSG_DELCHAIN: 580 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 581 continue; 582 583 policy = nft_trans_chain_policy(trans); 584 err = nft_flow_offload_chain(trans->ctx.chain, &policy, 585 FLOW_BLOCK_UNBIND); 586 break; 587 case NFT_MSG_NEWRULE: 588 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 589 continue; 590 591 if (trans->ctx.flags & NLM_F_REPLACE || 592 !(trans->ctx.flags & NLM_F_APPEND)) { 593 err = -EOPNOTSUPP; 594 break; 595 } 596 err = nft_flow_offload_rule(trans->ctx.chain, 597 nft_trans_rule(trans), 598 nft_trans_flow_rule(trans), 599 FLOW_CLS_REPLACE); 600 break; 601 case NFT_MSG_DELRULE: 602 if (!(trans->ctx.chain->flags & NFT_CHAIN_HW_OFFLOAD)) 603 continue; 604 605 err = nft_flow_offload_rule(trans->ctx.chain, 606 nft_trans_rule(trans), 607 NULL, FLOW_CLS_DESTROY); 608 break; 609 } 610 611 if (err) { 612 nft_flow_rule_offload_abort(net, trans); 613 break; 614 } 615 } 616 617 return err; 618 } 619 620 static struct nft_chain *__nft_offload_get_chain(const struct nftables_pernet *nft_net, 621 struct net_device *dev) 622 { 623 struct nft_base_chain *basechain; 624 struct nft_hook *hook, *found; 625 const struct nft_table *table; 626 struct nft_chain *chain; 627 628 list_for_each_entry(table, &nft_net->tables, list) { 629 if (table->family != NFPROTO_NETDEV) 630 continue; 631 632 list_for_each_entry(chain, &table->chains, list) { 633 if (!nft_is_base_chain(chain) || 634 !(chain->flags & NFT_CHAIN_HW_OFFLOAD)) 635 continue; 636 637 found = NULL; 638 basechain = nft_base_chain(chain); 639 list_for_each_entry(hook, &basechain->hook_list, list) { 640 if (hook->ops.dev != dev) 641 continue; 642 643 found = hook; 644 break; 645 } 646 if (!found) 647 continue; 648 649 return chain; 650 } 651 } 652 653 return NULL; 654 } 655 656 static int nft_offload_netdev_event(struct notifier_block *this, 657 unsigned long event, void *ptr) 658 { 659 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 660 struct nftables_pernet *nft_net; 661 struct net *net = dev_net(dev); 662 struct nft_chain *chain; 663 664 if (event != NETDEV_UNREGISTER) 665 return NOTIFY_DONE; 666 667 nft_net = nft_pernet(net); 668 mutex_lock(&nft_net->commit_mutex); 669 chain = __nft_offload_get_chain(nft_net, dev); 670 if (chain) 671 nft_flow_block_chain(nft_base_chain(chain), dev, 672 FLOW_BLOCK_UNBIND); 673 674 mutex_unlock(&nft_net->commit_mutex); 675 676 return NOTIFY_DONE; 677 } 678 679 static struct notifier_block nft_offload_netdev_notifier = { 680 .notifier_call = nft_offload_netdev_event, 681 }; 682 683 int nft_offload_init(void) 684 { 685 return register_netdevice_notifier(&nft_offload_netdev_notifier); 686 } 687 688 void nft_offload_exit(void) 689 { 690 unregister_netdevice_notifier(&nft_offload_netdev_notifier); 691 } 692