1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_pedit.c Generic packet editor 4 * 5 * Authors: Jamal Hadi Salim (2002-4) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/skbuff.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/slab.h> 19 #include <net/ipv6.h> 20 #include <net/netlink.h> 21 #include <net/pkt_sched.h> 22 #include <linux/tc_act/tc_pedit.h> 23 #include <net/tc_act/tc_pedit.h> 24 #include <uapi/linux/tc_act/tc_pedit.h> 25 #include <net/pkt_cls.h> 26 #include <net/tc_wrapper.h> 27 28 static struct tc_action_ops act_pedit_ops; 29 30 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { 31 [TCA_PEDIT_PARMS] = { .len = sizeof(struct tc_pedit) }, 32 [TCA_PEDIT_KEYS_EX] = { .type = NLA_NESTED }, 33 }; 34 35 static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = { 36 [TCA_PEDIT_KEY_EX_HTYPE] = 37 NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX), 38 [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX), 39 }; 40 41 static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla, 42 u8 n, struct netlink_ext_ack *extack) 43 { 44 struct tcf_pedit_key_ex *keys_ex; 45 struct tcf_pedit_key_ex *k; 46 const struct nlattr *ka; 47 int err = -EINVAL; 48 int rem; 49 50 if (!nla) 51 return NULL; 52 53 keys_ex = kcalloc(n, sizeof(*k), GFP_KERNEL); 54 if (!keys_ex) 55 return ERR_PTR(-ENOMEM); 56 57 k = keys_ex; 58 59 nla_for_each_nested(ka, nla, rem) { 60 struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1]; 61 62 if (!n) { 63 NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested"); 64 err = -EINVAL; 65 goto err_out; 66 } 67 n--; 68 69 if (nla_type(ka) != TCA_PEDIT_KEY_EX) { 70 NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key"); 71 err = -EINVAL; 72 goto err_out; 73 } 74 75 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX, 76 ka, pedit_key_ex_policy, 77 NULL); 78 if (err) 79 goto err_out; 80 81 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) { 82 NL_SET_ERR_MSG(extack, "Missing required attribute"); 83 err = -EINVAL; 84 goto err_out; 85 } 86 87 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) { 88 NL_SET_ERR_MSG(extack, "Missing required attribute"); 89 err = -EINVAL; 90 goto err_out; 91 } 92 93 k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]); 94 k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]); 95 96 k++; 97 } 98 99 if (n) { 100 NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse"); 101 err = -EINVAL; 102 goto err_out; 103 } 104 105 return keys_ex; 106 107 err_out: 108 kfree(keys_ex); 109 return ERR_PTR(err); 110 } 111 112 static int tcf_pedit_key_ex_dump(struct sk_buff *skb, 113 struct tcf_pedit_key_ex *keys_ex, int n) 114 { 115 struct nlattr *keys_start = nla_nest_start_noflag(skb, 116 TCA_PEDIT_KEYS_EX); 117 118 if (!keys_start) 119 goto nla_failure; 120 for (; n > 0; n--) { 121 struct nlattr *key_start; 122 123 key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX); 124 if (!key_start) 125 goto nla_failure; 126 127 if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) || 128 nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd)) 129 goto nla_failure; 130 131 nla_nest_end(skb, key_start); 132 133 keys_ex++; 134 } 135 136 nla_nest_end(skb, keys_start); 137 138 return 0; 139 nla_failure: 140 nla_nest_cancel(skb, keys_start); 141 return -EINVAL; 142 } 143 144 static void tcf_pedit_cleanup_rcu(struct rcu_head *head) 145 { 146 struct tcf_pedit_parms *parms = 147 container_of(head, struct tcf_pedit_parms, rcu); 148 149 kfree(parms->tcfp_keys_ex); 150 kfree(parms->tcfp_keys); 151 152 kfree(parms); 153 } 154 155 static int tcf_pedit_init(struct net *net, struct nlattr *nla, 156 struct nlattr *est, struct tc_action **a, 157 struct tcf_proto *tp, u32 flags, 158 struct netlink_ext_ack *extack) 159 { 160 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 161 bool bind = flags & TCA_ACT_FLAGS_BIND; 162 struct tcf_chain *goto_ch = NULL; 163 struct tcf_pedit_parms *oparms, *nparms; 164 struct nlattr *tb[TCA_PEDIT_MAX + 1]; 165 struct tc_pedit *parm; 166 struct nlattr *pattr; 167 struct tcf_pedit *p; 168 int ret = 0, err; 169 int i, ksize; 170 u32 index; 171 172 if (!nla) { 173 NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed"); 174 return -EINVAL; 175 } 176 177 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla, 178 pedit_policy, NULL); 179 if (err < 0) 180 return err; 181 182 pattr = tb[TCA_PEDIT_PARMS]; 183 if (!pattr) 184 pattr = tb[TCA_PEDIT_PARMS_EX]; 185 if (!pattr) { 186 NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute"); 187 return -EINVAL; 188 } 189 190 parm = nla_data(pattr); 191 192 index = parm->index; 193 err = tcf_idr_check_alloc(tn, &index, a, bind); 194 if (!err) { 195 ret = tcf_idr_create_from_flags(tn, index, est, a, 196 &act_pedit_ops, bind, flags); 197 if (ret) { 198 tcf_idr_cleanup(tn, index); 199 return ret; 200 } 201 ret = ACT_P_CREATED; 202 } else if (err > 0) { 203 if (bind) 204 return 0; 205 if (!(flags & TCA_ACT_FLAGS_REPLACE)) { 206 ret = -EEXIST; 207 goto out_release; 208 } 209 } else { 210 return err; 211 } 212 213 if (!parm->nkeys) { 214 NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed"); 215 ret = -EINVAL; 216 goto out_release; 217 } 218 ksize = parm->nkeys * sizeof(struct tc_pedit_key); 219 if (nla_len(pattr) < sizeof(*parm) + ksize) { 220 NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid"); 221 ret = -EINVAL; 222 goto out_release; 223 } 224 225 nparms = kzalloc(sizeof(*nparms), GFP_KERNEL); 226 if (!nparms) { 227 ret = -ENOMEM; 228 goto out_release; 229 } 230 231 nparms->tcfp_keys_ex = 232 tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack); 233 if (IS_ERR(nparms->tcfp_keys_ex)) { 234 ret = PTR_ERR(nparms->tcfp_keys_ex); 235 goto out_free; 236 } 237 238 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 239 if (err < 0) { 240 ret = err; 241 goto out_free_ex; 242 } 243 244 nparms->tcfp_off_max_hint = 0; 245 nparms->tcfp_flags = parm->flags; 246 nparms->tcfp_nkeys = parm->nkeys; 247 248 nparms->tcfp_keys = kmalloc(ksize, GFP_KERNEL); 249 if (!nparms->tcfp_keys) { 250 ret = -ENOMEM; 251 goto put_chain; 252 } 253 254 memcpy(nparms->tcfp_keys, parm->keys, ksize); 255 256 for (i = 0; i < nparms->tcfp_nkeys; ++i) { 257 u32 offmask = nparms->tcfp_keys[i].offmask; 258 u32 cur = nparms->tcfp_keys[i].off; 259 260 /* The AT option can be added to static offsets in the datapath */ 261 if (!offmask && cur % 4) { 262 NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries"); 263 ret = -EINVAL; 264 goto out_free_keys; 265 } 266 267 /* sanitize the shift value for any later use */ 268 nparms->tcfp_keys[i].shift = min_t(size_t, 269 BITS_PER_TYPE(int) - 1, 270 nparms->tcfp_keys[i].shift); 271 272 /* The AT option can read a single byte, we can bound the actual 273 * value with uchar max. 274 */ 275 cur += (0xff & offmask) >> nparms->tcfp_keys[i].shift; 276 277 /* Each key touches 4 bytes starting from the computed offset */ 278 nparms->tcfp_off_max_hint = 279 max(nparms->tcfp_off_max_hint, cur + 4); 280 } 281 282 p = to_pedit(*a); 283 284 spin_lock_bh(&p->tcf_lock); 285 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 286 oparms = rcu_replace_pointer(p->parms, nparms, 1); 287 spin_unlock_bh(&p->tcf_lock); 288 289 if (oparms) 290 call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu); 291 292 if (goto_ch) 293 tcf_chain_put_by_act(goto_ch); 294 295 return ret; 296 297 out_free_keys: 298 kfree(nparms->tcfp_keys); 299 put_chain: 300 if (goto_ch) 301 tcf_chain_put_by_act(goto_ch); 302 out_free_ex: 303 kfree(nparms->tcfp_keys_ex); 304 out_free: 305 kfree(nparms); 306 out_release: 307 tcf_idr_release(*a, bind); 308 return ret; 309 } 310 311 static void tcf_pedit_cleanup(struct tc_action *a) 312 { 313 struct tcf_pedit *p = to_pedit(a); 314 struct tcf_pedit_parms *parms; 315 316 parms = rcu_dereference_protected(p->parms, 1); 317 318 if (parms) 319 call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu); 320 } 321 322 static bool offset_valid(struct sk_buff *skb, int offset) 323 { 324 if (offset > 0 && offset > skb->len) 325 return false; 326 327 if (offset < 0 && -offset > skb_headroom(skb)) 328 return false; 329 330 return true; 331 } 332 333 static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type) 334 { 335 const int noff = skb_network_offset(skb); 336 int ret = -EINVAL; 337 struct iphdr _iph; 338 339 switch (skb->protocol) { 340 case htons(ETH_P_IP): { 341 const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph); 342 343 if (!iph) 344 goto out; 345 *hoffset = noff + iph->ihl * 4; 346 ret = 0; 347 break; 348 } 349 case htons(ETH_P_IPV6): 350 ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL; 351 break; 352 } 353 out: 354 return ret; 355 } 356 357 static int pedit_skb_hdr_offset(struct sk_buff *skb, 358 enum pedit_header_type htype, int *hoffset) 359 { 360 int ret = -EINVAL; 361 /* 'htype' is validated in the netlink parsing */ 362 switch (htype) { 363 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: 364 if (skb_mac_header_was_set(skb)) { 365 *hoffset = skb_mac_offset(skb); 366 ret = 0; 367 } 368 break; 369 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK: 370 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: 371 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: 372 *hoffset = skb_network_offset(skb); 373 ret = 0; 374 break; 375 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: 376 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP); 377 break; 378 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: 379 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP); 380 break; 381 default: 382 break; 383 } 384 return ret; 385 } 386 387 TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, 388 const struct tc_action *a, 389 struct tcf_result *res) 390 { 391 enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK; 392 enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET; 393 struct tcf_pedit *p = to_pedit(a); 394 struct tcf_pedit_key_ex *tkey_ex; 395 struct tcf_pedit_parms *parms; 396 struct tc_pedit_key *tkey; 397 u32 max_offset; 398 int i; 399 400 parms = rcu_dereference_bh(p->parms); 401 402 max_offset = (skb_transport_header_was_set(skb) ? 403 skb_transport_offset(skb) : 404 skb_network_offset(skb)) + 405 parms->tcfp_off_max_hint; 406 if (skb_ensure_writable(skb, min(skb->len, max_offset))) 407 goto done; 408 409 tcf_lastuse_update(&p->tcf_tm); 410 tcf_action_update_bstats(&p->common, skb); 411 412 tkey = parms->tcfp_keys; 413 tkey_ex = parms->tcfp_keys_ex; 414 415 for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) { 416 int offset = tkey->off; 417 int hoffset = 0; 418 u32 *ptr, hdata; 419 u32 val; 420 int rc; 421 422 if (tkey_ex) { 423 htype = tkey_ex->htype; 424 cmd = tkey_ex->cmd; 425 426 tkey_ex++; 427 } 428 429 rc = pedit_skb_hdr_offset(skb, htype, &hoffset); 430 if (rc) { 431 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype); 432 goto bad; 433 } 434 435 if (tkey->offmask) { 436 u8 *d, _d; 437 438 if (!offset_valid(skb, hoffset + tkey->at)) { 439 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n", 440 hoffset + tkey->at); 441 goto bad; 442 } 443 d = skb_header_pointer(skb, hoffset + tkey->at, 444 sizeof(_d), &_d); 445 if (!d) 446 goto bad; 447 448 offset += (*d & tkey->offmask) >> tkey->shift; 449 if (offset % 4) { 450 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n"); 451 goto bad; 452 } 453 } 454 455 if (!offset_valid(skb, hoffset + offset)) { 456 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset + offset); 457 goto bad; 458 } 459 460 ptr = skb_header_pointer(skb, hoffset + offset, 461 sizeof(hdata), &hdata); 462 if (!ptr) 463 goto bad; 464 /* just do it, baby */ 465 switch (cmd) { 466 case TCA_PEDIT_KEY_EX_CMD_SET: 467 val = tkey->val; 468 break; 469 case TCA_PEDIT_KEY_EX_CMD_ADD: 470 val = (*ptr + tkey->val) & ~tkey->mask; 471 break; 472 default: 473 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd); 474 goto bad; 475 } 476 477 *ptr = ((*ptr & tkey->mask) ^ val); 478 if (ptr == &hdata) 479 skb_store_bits(skb, hoffset + offset, ptr, 4); 480 } 481 482 goto done; 483 484 bad: 485 tcf_action_inc_overlimit_qstats(&p->common); 486 done: 487 return p->tcf_action; 488 } 489 490 static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, 491 u64 drops, u64 lastuse, bool hw) 492 { 493 struct tcf_pedit *d = to_pedit(a); 494 struct tcf_t *tm = &d->tcf_tm; 495 496 tcf_action_update_stats(a, bytes, packets, drops, hw); 497 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 498 } 499 500 static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, 501 int bind, int ref) 502 { 503 unsigned char *b = skb_tail_pointer(skb); 504 struct tcf_pedit *p = to_pedit(a); 505 struct tcf_pedit_parms *parms; 506 struct tc_pedit *opt; 507 struct tcf_t t; 508 int s; 509 510 spin_lock_bh(&p->tcf_lock); 511 parms = rcu_dereference_protected(p->parms, 1); 512 s = struct_size(opt, keys, parms->tcfp_nkeys); 513 514 opt = kzalloc(s, GFP_ATOMIC); 515 if (unlikely(!opt)) { 516 spin_unlock_bh(&p->tcf_lock); 517 return -ENOBUFS; 518 } 519 520 memcpy(opt->keys, parms->tcfp_keys, 521 flex_array_size(opt, keys, parms->tcfp_nkeys)); 522 opt->index = p->tcf_index; 523 opt->nkeys = parms->tcfp_nkeys; 524 opt->flags = parms->tcfp_flags; 525 opt->action = p->tcf_action; 526 opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; 527 opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; 528 529 if (parms->tcfp_keys_ex) { 530 if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex, 531 parms->tcfp_nkeys)) 532 goto nla_put_failure; 533 534 if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt)) 535 goto nla_put_failure; 536 } else { 537 if (nla_put(skb, TCA_PEDIT_PARMS, s, opt)) 538 goto nla_put_failure; 539 } 540 541 tcf_tm_dump(&t, &p->tcf_tm); 542 if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) 543 goto nla_put_failure; 544 spin_unlock_bh(&p->tcf_lock); 545 546 kfree(opt); 547 return skb->len; 548 549 nla_put_failure: 550 spin_unlock_bh(&p->tcf_lock); 551 nlmsg_trim(skb, b); 552 kfree(opt); 553 return -1; 554 } 555 556 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, 557 u32 *index_inc, bool bind, 558 struct netlink_ext_ack *extack) 559 { 560 if (bind) { 561 struct flow_action_entry *entry = entry_data; 562 int k; 563 564 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 565 switch (tcf_pedit_cmd(act, k)) { 566 case TCA_PEDIT_KEY_EX_CMD_SET: 567 entry->id = FLOW_ACTION_MANGLE; 568 break; 569 case TCA_PEDIT_KEY_EX_CMD_ADD: 570 entry->id = FLOW_ACTION_ADD; 571 break; 572 default: 573 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 574 return -EOPNOTSUPP; 575 } 576 entry->mangle.htype = tcf_pedit_htype(act, k); 577 entry->mangle.mask = tcf_pedit_mask(act, k); 578 entry->mangle.val = tcf_pedit_val(act, k); 579 entry->mangle.offset = tcf_pedit_offset(act, k); 580 entry->hw_stats = tc_act_hw_stats(act->hw_stats); 581 entry++; 582 } 583 *index_inc = k; 584 } else { 585 struct flow_offload_action *fl_action = entry_data; 586 u32 cmd = tcf_pedit_cmd(act, 0); 587 int k; 588 589 switch (cmd) { 590 case TCA_PEDIT_KEY_EX_CMD_SET: 591 fl_action->id = FLOW_ACTION_MANGLE; 592 break; 593 case TCA_PEDIT_KEY_EX_CMD_ADD: 594 fl_action->id = FLOW_ACTION_ADD; 595 break; 596 default: 597 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 598 return -EOPNOTSUPP; 599 } 600 601 for (k = 1; k < tcf_pedit_nkeys(act); k++) { 602 if (cmd != tcf_pedit_cmd(act, k)) { 603 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 604 return -EOPNOTSUPP; 605 } 606 } 607 } 608 609 return 0; 610 } 611 612 static struct tc_action_ops act_pedit_ops = { 613 .kind = "pedit", 614 .id = TCA_ID_PEDIT, 615 .owner = THIS_MODULE, 616 .act = tcf_pedit_act, 617 .stats_update = tcf_pedit_stats_update, 618 .dump = tcf_pedit_dump, 619 .cleanup = tcf_pedit_cleanup, 620 .init = tcf_pedit_init, 621 .offload_act_setup = tcf_pedit_offload_act_setup, 622 .size = sizeof(struct tcf_pedit), 623 }; 624 625 static __net_init int pedit_init_net(struct net *net) 626 { 627 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 628 629 return tc_action_net_init(net, tn, &act_pedit_ops); 630 } 631 632 static void __net_exit pedit_exit_net(struct list_head *net_list) 633 { 634 tc_action_net_exit(net_list, act_pedit_ops.net_id); 635 } 636 637 static struct pernet_operations pedit_net_ops = { 638 .init = pedit_init_net, 639 .exit_batch = pedit_exit_net, 640 .id = &act_pedit_ops.net_id, 641 .size = sizeof(struct tc_action_net), 642 }; 643 644 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)"); 645 MODULE_DESCRIPTION("Generic Packet Editor actions"); 646 MODULE_LICENSE("GPL"); 647 648 static int __init pedit_init_module(void) 649 { 650 return tcf_register_action(&act_pedit_ops, &pedit_net_ops); 651 } 652 653 static void __exit pedit_cleanup_module(void) 654 { 655 tcf_unregister_action(&act_pedit_ops, &pedit_net_ops); 656 } 657 658 module_init(pedit_init_module); 659 module_exit(pedit_cleanup_module); 660