1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_pedit.c Generic packet editor 4 * 5 * Authors: Jamal Hadi Salim (2002-4) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/skbuff.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/slab.h> 19 #include <net/ipv6.h> 20 #include <net/netlink.h> 21 #include <net/pkt_sched.h> 22 #include <linux/tc_act/tc_pedit.h> 23 #include <net/tc_act/tc_pedit.h> 24 #include <uapi/linux/tc_act/tc_pedit.h> 25 #include <net/pkt_cls.h> 26 #include <net/tc_wrapper.h> 27 28 static struct tc_action_ops act_pedit_ops; 29 30 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { 31 [TCA_PEDIT_PARMS] = { .len = sizeof(struct tc_pedit) }, 32 [TCA_PEDIT_KEYS_EX] = { .type = NLA_NESTED }, 33 }; 34 35 static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = { 36 [TCA_PEDIT_KEY_EX_HTYPE] = 37 NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX), 38 [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX), 39 }; 40 41 static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla, 42 u8 n, struct netlink_ext_ack *extack) 43 { 44 struct tcf_pedit_key_ex *keys_ex; 45 struct tcf_pedit_key_ex *k; 46 const struct nlattr *ka; 47 int err = -EINVAL; 48 int rem; 49 50 if (!nla) 51 return NULL; 52 53 keys_ex = kcalloc(n, sizeof(*k), GFP_KERNEL); 54 if (!keys_ex) 55 return ERR_PTR(-ENOMEM); 56 57 k = keys_ex; 58 59 nla_for_each_nested(ka, nla, rem) { 60 struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1]; 61 62 if (!n) { 63 NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested"); 64 err = -EINVAL; 65 goto err_out; 66 } 67 n--; 68 69 if (nla_type(ka) != TCA_PEDIT_KEY_EX) { 70 NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key"); 71 err = -EINVAL; 72 goto err_out; 73 } 74 75 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX, 76 ka, pedit_key_ex_policy, 77 NULL); 78 if (err) 79 goto err_out; 80 81 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) { 82 NL_SET_ERR_MSG(extack, "Missing required attribute"); 83 err = -EINVAL; 84 goto err_out; 85 } 86 87 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) { 88 NL_SET_ERR_MSG(extack, "Missing required attribute"); 89 err = -EINVAL; 90 goto err_out; 91 } 92 93 k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]); 94 k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]); 95 96 k++; 97 } 98 99 if (n) { 100 NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse"); 101 err = -EINVAL; 102 goto err_out; 103 } 104 105 return keys_ex; 106 107 err_out: 108 kfree(keys_ex); 109 return ERR_PTR(err); 110 } 111 112 static int tcf_pedit_key_ex_dump(struct sk_buff *skb, 113 struct tcf_pedit_key_ex *keys_ex, int n) 114 { 115 struct nlattr *keys_start = nla_nest_start_noflag(skb, 116 TCA_PEDIT_KEYS_EX); 117 118 if (!keys_start) 119 goto nla_failure; 120 for (; n > 0; n--) { 121 struct nlattr *key_start; 122 123 key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX); 124 if (!key_start) 125 goto nla_failure; 126 127 if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) || 128 nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd)) 129 goto nla_failure; 130 131 nla_nest_end(skb, key_start); 132 133 keys_ex++; 134 } 135 136 nla_nest_end(skb, keys_start); 137 138 return 0; 139 nla_failure: 140 nla_nest_cancel(skb, keys_start); 141 return -EINVAL; 142 } 143 144 static void tcf_pedit_cleanup_rcu(struct rcu_head *head) 145 { 146 struct tcf_pedit_parms *parms = 147 container_of(head, struct tcf_pedit_parms, rcu); 148 149 kfree(parms->tcfp_keys_ex); 150 kfree(parms->tcfp_keys); 151 152 kfree(parms); 153 } 154 155 static int tcf_pedit_init(struct net *net, struct nlattr *nla, 156 struct nlattr *est, struct tc_action **a, 157 struct tcf_proto *tp, u32 flags, 158 struct netlink_ext_ack *extack) 159 { 160 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 161 bool bind = flags & TCA_ACT_FLAGS_BIND; 162 struct tcf_chain *goto_ch = NULL; 163 struct tcf_pedit_parms *oparms, *nparms; 164 struct nlattr *tb[TCA_PEDIT_MAX + 1]; 165 struct tc_pedit *parm; 166 struct nlattr *pattr; 167 struct tcf_pedit *p; 168 int ret = 0, err; 169 int i, ksize; 170 u32 index; 171 172 if (!nla) { 173 NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed"); 174 return -EINVAL; 175 } 176 177 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla, 178 pedit_policy, NULL); 179 if (err < 0) 180 return err; 181 182 pattr = tb[TCA_PEDIT_PARMS]; 183 if (!pattr) 184 pattr = tb[TCA_PEDIT_PARMS_EX]; 185 if (!pattr) { 186 NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute"); 187 return -EINVAL; 188 } 189 190 parm = nla_data(pattr); 191 192 index = parm->index; 193 err = tcf_idr_check_alloc(tn, &index, a, bind); 194 if (!err) { 195 ret = tcf_idr_create_from_flags(tn, index, est, a, 196 &act_pedit_ops, bind, flags); 197 if (ret) { 198 tcf_idr_cleanup(tn, index); 199 return ret; 200 } 201 ret = ACT_P_CREATED; 202 } else if (err > 0) { 203 if (bind) 204 return 0; 205 if (!(flags & TCA_ACT_FLAGS_REPLACE)) { 206 ret = -EEXIST; 207 goto out_release; 208 } 209 } else { 210 return err; 211 } 212 213 if (!parm->nkeys) { 214 NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed"); 215 ret = -EINVAL; 216 goto out_release; 217 } 218 ksize = parm->nkeys * sizeof(struct tc_pedit_key); 219 if (nla_len(pattr) < sizeof(*parm) + ksize) { 220 NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid"); 221 ret = -EINVAL; 222 goto out_release; 223 } 224 225 nparms = kzalloc(sizeof(*nparms), GFP_KERNEL); 226 if (!nparms) { 227 ret = -ENOMEM; 228 goto out_release; 229 } 230 231 nparms->tcfp_keys_ex = 232 tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack); 233 if (IS_ERR(nparms->tcfp_keys_ex)) { 234 ret = PTR_ERR(nparms->tcfp_keys_ex); 235 goto out_free; 236 } 237 238 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 239 if (err < 0) { 240 ret = err; 241 goto out_free_ex; 242 } 243 244 nparms->tcfp_off_max_hint = 0; 245 nparms->tcfp_flags = parm->flags; 246 nparms->tcfp_nkeys = parm->nkeys; 247 248 nparms->tcfp_keys = kmemdup(parm->keys, ksize, GFP_KERNEL); 249 if (!nparms->tcfp_keys) { 250 ret = -ENOMEM; 251 goto put_chain; 252 } 253 254 for (i = 0; i < nparms->tcfp_nkeys; ++i) { 255 u32 offmask = nparms->tcfp_keys[i].offmask; 256 u32 cur = nparms->tcfp_keys[i].off; 257 258 /* The AT option can be added to static offsets in the datapath */ 259 if (!offmask && cur % 4) { 260 NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries"); 261 ret = -EINVAL; 262 goto out_free_keys; 263 } 264 265 /* sanitize the shift value for any later use */ 266 nparms->tcfp_keys[i].shift = min_t(size_t, 267 BITS_PER_TYPE(int) - 1, 268 nparms->tcfp_keys[i].shift); 269 270 /* The AT option can read a single byte, we can bound the actual 271 * value with uchar max. 272 */ 273 cur += (0xff & offmask) >> nparms->tcfp_keys[i].shift; 274 275 /* Each key touches 4 bytes starting from the computed offset */ 276 nparms->tcfp_off_max_hint = 277 max(nparms->tcfp_off_max_hint, cur + 4); 278 } 279 280 p = to_pedit(*a); 281 282 spin_lock_bh(&p->tcf_lock); 283 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 284 oparms = rcu_replace_pointer(p->parms, nparms, 1); 285 spin_unlock_bh(&p->tcf_lock); 286 287 if (oparms) 288 call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu); 289 290 if (goto_ch) 291 tcf_chain_put_by_act(goto_ch); 292 293 return ret; 294 295 out_free_keys: 296 kfree(nparms->tcfp_keys); 297 put_chain: 298 if (goto_ch) 299 tcf_chain_put_by_act(goto_ch); 300 out_free_ex: 301 kfree(nparms->tcfp_keys_ex); 302 out_free: 303 kfree(nparms); 304 out_release: 305 tcf_idr_release(*a, bind); 306 return ret; 307 } 308 309 static void tcf_pedit_cleanup(struct tc_action *a) 310 { 311 struct tcf_pedit *p = to_pedit(a); 312 struct tcf_pedit_parms *parms; 313 314 parms = rcu_dereference_protected(p->parms, 1); 315 316 if (parms) 317 call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu); 318 } 319 320 static bool offset_valid(struct sk_buff *skb, int offset) 321 { 322 if (offset > 0 && offset > skb->len) 323 return false; 324 325 if (offset < 0 && -offset > skb_headroom(skb)) 326 return false; 327 328 return true; 329 } 330 331 static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type) 332 { 333 const int noff = skb_network_offset(skb); 334 int ret = -EINVAL; 335 struct iphdr _iph; 336 337 switch (skb->protocol) { 338 case htons(ETH_P_IP): { 339 const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph); 340 341 if (!iph) 342 goto out; 343 *hoffset = noff + iph->ihl * 4; 344 ret = 0; 345 break; 346 } 347 case htons(ETH_P_IPV6): 348 ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL; 349 break; 350 } 351 out: 352 return ret; 353 } 354 355 static int pedit_skb_hdr_offset(struct sk_buff *skb, 356 enum pedit_header_type htype, int *hoffset) 357 { 358 int ret = -EINVAL; 359 /* 'htype' is validated in the netlink parsing */ 360 switch (htype) { 361 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: 362 if (skb_mac_header_was_set(skb)) { 363 *hoffset = skb_mac_offset(skb); 364 ret = 0; 365 } 366 break; 367 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK: 368 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: 369 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: 370 *hoffset = skb_network_offset(skb); 371 ret = 0; 372 break; 373 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: 374 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP); 375 break; 376 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: 377 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP); 378 break; 379 default: 380 break; 381 } 382 return ret; 383 } 384 385 TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, 386 const struct tc_action *a, 387 struct tcf_result *res) 388 { 389 enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK; 390 enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET; 391 struct tcf_pedit *p = to_pedit(a); 392 struct tcf_pedit_key_ex *tkey_ex; 393 struct tcf_pedit_parms *parms; 394 struct tc_pedit_key *tkey; 395 u32 max_offset; 396 int i; 397 398 parms = rcu_dereference_bh(p->parms); 399 400 max_offset = (skb_transport_header_was_set(skb) ? 401 skb_transport_offset(skb) : 402 skb_network_offset(skb)) + 403 parms->tcfp_off_max_hint; 404 if (skb_ensure_writable(skb, min(skb->len, max_offset))) 405 goto done; 406 407 tcf_lastuse_update(&p->tcf_tm); 408 tcf_action_update_bstats(&p->common, skb); 409 410 tkey = parms->tcfp_keys; 411 tkey_ex = parms->tcfp_keys_ex; 412 413 for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) { 414 int offset = tkey->off; 415 int hoffset = 0; 416 u32 *ptr, hdata; 417 u32 val; 418 int rc; 419 420 if (tkey_ex) { 421 htype = tkey_ex->htype; 422 cmd = tkey_ex->cmd; 423 424 tkey_ex++; 425 } 426 427 rc = pedit_skb_hdr_offset(skb, htype, &hoffset); 428 if (rc) { 429 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype); 430 goto bad; 431 } 432 433 if (tkey->offmask) { 434 u8 *d, _d; 435 436 if (!offset_valid(skb, hoffset + tkey->at)) { 437 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n", 438 hoffset + tkey->at); 439 goto bad; 440 } 441 d = skb_header_pointer(skb, hoffset + tkey->at, 442 sizeof(_d), &_d); 443 if (!d) 444 goto bad; 445 446 offset += (*d & tkey->offmask) >> tkey->shift; 447 if (offset % 4) { 448 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n"); 449 goto bad; 450 } 451 } 452 453 if (!offset_valid(skb, hoffset + offset)) { 454 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset + offset); 455 goto bad; 456 } 457 458 ptr = skb_header_pointer(skb, hoffset + offset, 459 sizeof(hdata), &hdata); 460 if (!ptr) 461 goto bad; 462 /* just do it, baby */ 463 switch (cmd) { 464 case TCA_PEDIT_KEY_EX_CMD_SET: 465 val = tkey->val; 466 break; 467 case TCA_PEDIT_KEY_EX_CMD_ADD: 468 val = (*ptr + tkey->val) & ~tkey->mask; 469 break; 470 default: 471 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd); 472 goto bad; 473 } 474 475 *ptr = ((*ptr & tkey->mask) ^ val); 476 if (ptr == &hdata) 477 skb_store_bits(skb, hoffset + offset, ptr, 4); 478 } 479 480 goto done; 481 482 bad: 483 tcf_action_inc_overlimit_qstats(&p->common); 484 done: 485 return p->tcf_action; 486 } 487 488 static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, 489 u64 drops, u64 lastuse, bool hw) 490 { 491 struct tcf_pedit *d = to_pedit(a); 492 struct tcf_t *tm = &d->tcf_tm; 493 494 tcf_action_update_stats(a, bytes, packets, drops, hw); 495 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 496 } 497 498 static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, 499 int bind, int ref) 500 { 501 unsigned char *b = skb_tail_pointer(skb); 502 struct tcf_pedit *p = to_pedit(a); 503 struct tcf_pedit_parms *parms; 504 struct tc_pedit *opt; 505 struct tcf_t t; 506 int s; 507 508 spin_lock_bh(&p->tcf_lock); 509 parms = rcu_dereference_protected(p->parms, 1); 510 s = struct_size(opt, keys, parms->tcfp_nkeys); 511 512 opt = kzalloc(s, GFP_ATOMIC); 513 if (unlikely(!opt)) { 514 spin_unlock_bh(&p->tcf_lock); 515 return -ENOBUFS; 516 } 517 518 memcpy(opt->keys, parms->tcfp_keys, 519 flex_array_size(opt, keys, parms->tcfp_nkeys)); 520 opt->index = p->tcf_index; 521 opt->nkeys = parms->tcfp_nkeys; 522 opt->flags = parms->tcfp_flags; 523 opt->action = p->tcf_action; 524 opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; 525 opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; 526 527 if (parms->tcfp_keys_ex) { 528 if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex, 529 parms->tcfp_nkeys)) 530 goto nla_put_failure; 531 532 if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt)) 533 goto nla_put_failure; 534 } else { 535 if (nla_put(skb, TCA_PEDIT_PARMS, s, opt)) 536 goto nla_put_failure; 537 } 538 539 tcf_tm_dump(&t, &p->tcf_tm); 540 if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) 541 goto nla_put_failure; 542 spin_unlock_bh(&p->tcf_lock); 543 544 kfree(opt); 545 return skb->len; 546 547 nla_put_failure: 548 spin_unlock_bh(&p->tcf_lock); 549 nlmsg_trim(skb, b); 550 kfree(opt); 551 return -1; 552 } 553 554 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, 555 u32 *index_inc, bool bind, 556 struct netlink_ext_ack *extack) 557 { 558 if (bind) { 559 struct flow_action_entry *entry = entry_data; 560 int k; 561 562 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 563 switch (tcf_pedit_cmd(act, k)) { 564 case TCA_PEDIT_KEY_EX_CMD_SET: 565 entry->id = FLOW_ACTION_MANGLE; 566 break; 567 case TCA_PEDIT_KEY_EX_CMD_ADD: 568 entry->id = FLOW_ACTION_ADD; 569 break; 570 default: 571 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 572 return -EOPNOTSUPP; 573 } 574 entry->mangle.htype = tcf_pedit_htype(act, k); 575 entry->mangle.mask = tcf_pedit_mask(act, k); 576 entry->mangle.val = tcf_pedit_val(act, k); 577 entry->mangle.offset = tcf_pedit_offset(act, k); 578 entry->hw_stats = tc_act_hw_stats(act->hw_stats); 579 entry++; 580 } 581 *index_inc = k; 582 } else { 583 struct flow_offload_action *fl_action = entry_data; 584 u32 cmd = tcf_pedit_cmd(act, 0); 585 int k; 586 587 switch (cmd) { 588 case TCA_PEDIT_KEY_EX_CMD_SET: 589 fl_action->id = FLOW_ACTION_MANGLE; 590 break; 591 case TCA_PEDIT_KEY_EX_CMD_ADD: 592 fl_action->id = FLOW_ACTION_ADD; 593 break; 594 default: 595 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 596 return -EOPNOTSUPP; 597 } 598 599 for (k = 1; k < tcf_pedit_nkeys(act); k++) { 600 if (cmd != tcf_pedit_cmd(act, k)) { 601 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 602 return -EOPNOTSUPP; 603 } 604 } 605 } 606 607 return 0; 608 } 609 610 static struct tc_action_ops act_pedit_ops = { 611 .kind = "pedit", 612 .id = TCA_ID_PEDIT, 613 .owner = THIS_MODULE, 614 .act = tcf_pedit_act, 615 .stats_update = tcf_pedit_stats_update, 616 .dump = tcf_pedit_dump, 617 .cleanup = tcf_pedit_cleanup, 618 .init = tcf_pedit_init, 619 .offload_act_setup = tcf_pedit_offload_act_setup, 620 .size = sizeof(struct tcf_pedit), 621 }; 622 623 static __net_init int pedit_init_net(struct net *net) 624 { 625 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 626 627 return tc_action_net_init(net, tn, &act_pedit_ops); 628 } 629 630 static void __net_exit pedit_exit_net(struct list_head *net_list) 631 { 632 tc_action_net_exit(net_list, act_pedit_ops.net_id); 633 } 634 635 static struct pernet_operations pedit_net_ops = { 636 .init = pedit_init_net, 637 .exit_batch = pedit_exit_net, 638 .id = &act_pedit_ops.net_id, 639 .size = sizeof(struct tc_action_net), 640 }; 641 642 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)"); 643 MODULE_DESCRIPTION("Generic Packet Editor actions"); 644 MODULE_LICENSE("GPL"); 645 646 static int __init pedit_init_module(void) 647 { 648 return tcf_register_action(&act_pedit_ops, &pedit_net_ops); 649 } 650 651 static void __exit pedit_cleanup_module(void) 652 { 653 tcf_unregister_action(&act_pedit_ops, &pedit_net_ops); 654 } 655 656 module_init(pedit_init_module); 657 module_exit(pedit_cleanup_module); 658