1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008, Intel Corporation. 4 * 5 * Author: Alexander Duyck <alexander.h.duyck@intel.com> 6 */ 7 8 #include <linux/module.h> 9 #include <linux/init.h> 10 #include <linux/kernel.h> 11 #include <linux/skbuff.h> 12 #include <linux/rtnetlink.h> 13 #include <net/netlink.h> 14 #include <net/pkt_sched.h> 15 #include <net/ip.h> 16 #include <net/ipv6.h> 17 #include <net/dsfield.h> 18 #include <net/pkt_cls.h> 19 20 #include <linux/tc_act/tc_skbedit.h> 21 #include <net/tc_act/tc_skbedit.h> 22 23 static unsigned int skbedit_net_id; 24 static struct tc_action_ops act_skbedit_ops; 25 26 static u16 tcf_skbedit_hash(struct tcf_skbedit_params *params, 27 struct sk_buff *skb) 28 { 29 u16 queue_mapping = params->queue_mapping; 30 31 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) { 32 u32 hash = skb_get_hash(skb); 33 34 queue_mapping += hash % params->mapping_mod; 35 } 36 37 return netdev_cap_txqueue(skb->dev, queue_mapping); 38 } 39 40 static int tcf_skbedit_act(struct sk_buff *skb, const struct tc_action *a, 41 struct tcf_result *res) 42 { 43 struct tcf_skbedit *d = to_skbedit(a); 44 struct tcf_skbedit_params *params; 45 int action; 46 47 tcf_lastuse_update(&d->tcf_tm); 48 bstats_update(this_cpu_ptr(d->common.cpu_bstats), skb); 49 50 params = rcu_dereference_bh(d->params); 51 action = READ_ONCE(d->tcf_action); 52 53 if (params->flags & SKBEDIT_F_PRIORITY) 54 skb->priority = params->priority; 55 if (params->flags & SKBEDIT_F_INHERITDSFIELD) { 56 int wlen = skb_network_offset(skb); 57 58 switch (skb_protocol(skb, true)) { 59 case htons(ETH_P_IP): 60 wlen += sizeof(struct iphdr); 61 if (!pskb_may_pull(skb, wlen)) 62 goto err; 63 skb->priority = ipv4_get_dsfield(ip_hdr(skb)) >> 2; 64 break; 65 66 case htons(ETH_P_IPV6): 67 wlen += sizeof(struct ipv6hdr); 68 if (!pskb_may_pull(skb, wlen)) 69 goto err; 70 skb->priority = ipv6_get_dsfield(ipv6_hdr(skb)) >> 2; 71 break; 72 } 73 } 74 if (params->flags & SKBEDIT_F_QUEUE_MAPPING && 75 skb->dev->real_num_tx_queues > params->queue_mapping) { 76 #ifdef CONFIG_NET_EGRESS 77 netdev_xmit_skip_txqueue(true); 78 #endif 79 skb_set_queue_mapping(skb, tcf_skbedit_hash(params, skb)); 80 } 81 if (params->flags & SKBEDIT_F_MARK) { 82 skb->mark &= ~params->mask; 83 skb->mark |= params->mark & params->mask; 84 } 85 if (params->flags & SKBEDIT_F_PTYPE) 86 skb->pkt_type = params->ptype; 87 return action; 88 89 err: 90 qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats)); 91 return TC_ACT_SHOT; 92 } 93 94 static void tcf_skbedit_stats_update(struct tc_action *a, u64 bytes, 95 u64 packets, u64 drops, 96 u64 lastuse, bool hw) 97 { 98 struct tcf_skbedit *d = to_skbedit(a); 99 struct tcf_t *tm = &d->tcf_tm; 100 101 tcf_action_update_stats(a, bytes, packets, drops, hw); 102 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 103 } 104 105 static const struct nla_policy skbedit_policy[TCA_SKBEDIT_MAX + 1] = { 106 [TCA_SKBEDIT_PARMS] = { .len = sizeof(struct tc_skbedit) }, 107 [TCA_SKBEDIT_PRIORITY] = { .len = sizeof(u32) }, 108 [TCA_SKBEDIT_QUEUE_MAPPING] = { .len = sizeof(u16) }, 109 [TCA_SKBEDIT_MARK] = { .len = sizeof(u32) }, 110 [TCA_SKBEDIT_PTYPE] = { .len = sizeof(u16) }, 111 [TCA_SKBEDIT_MASK] = { .len = sizeof(u32) }, 112 [TCA_SKBEDIT_FLAGS] = { .len = sizeof(u64) }, 113 [TCA_SKBEDIT_QUEUE_MAPPING_MAX] = { .len = sizeof(u16) }, 114 }; 115 116 static int tcf_skbedit_init(struct net *net, struct nlattr *nla, 117 struct nlattr *est, struct tc_action **a, 118 struct tcf_proto *tp, u32 act_flags, 119 struct netlink_ext_ack *extack) 120 { 121 struct tc_action_net *tn = net_generic(net, skbedit_net_id); 122 bool bind = act_flags & TCA_ACT_FLAGS_BIND; 123 struct tcf_skbedit_params *params_new; 124 struct nlattr *tb[TCA_SKBEDIT_MAX + 1]; 125 struct tcf_chain *goto_ch = NULL; 126 struct tc_skbedit *parm; 127 struct tcf_skbedit *d; 128 u32 flags = 0, *priority = NULL, *mark = NULL, *mask = NULL; 129 u16 *queue_mapping = NULL, *ptype = NULL; 130 u16 mapping_mod = 1; 131 bool exists = false; 132 int ret = 0, err; 133 u32 index; 134 135 if (nla == NULL) 136 return -EINVAL; 137 138 err = nla_parse_nested_deprecated(tb, TCA_SKBEDIT_MAX, nla, 139 skbedit_policy, NULL); 140 if (err < 0) 141 return err; 142 143 if (tb[TCA_SKBEDIT_PARMS] == NULL) 144 return -EINVAL; 145 146 if (tb[TCA_SKBEDIT_PRIORITY] != NULL) { 147 flags |= SKBEDIT_F_PRIORITY; 148 priority = nla_data(tb[TCA_SKBEDIT_PRIORITY]); 149 } 150 151 if (tb[TCA_SKBEDIT_QUEUE_MAPPING] != NULL) { 152 flags |= SKBEDIT_F_QUEUE_MAPPING; 153 queue_mapping = nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING]); 154 } 155 156 if (tb[TCA_SKBEDIT_PTYPE] != NULL) { 157 ptype = nla_data(tb[TCA_SKBEDIT_PTYPE]); 158 if (!skb_pkt_type_ok(*ptype)) 159 return -EINVAL; 160 flags |= SKBEDIT_F_PTYPE; 161 } 162 163 if (tb[TCA_SKBEDIT_MARK] != NULL) { 164 flags |= SKBEDIT_F_MARK; 165 mark = nla_data(tb[TCA_SKBEDIT_MARK]); 166 } 167 168 if (tb[TCA_SKBEDIT_MASK] != NULL) { 169 flags |= SKBEDIT_F_MASK; 170 mask = nla_data(tb[TCA_SKBEDIT_MASK]); 171 } 172 173 if (tb[TCA_SKBEDIT_FLAGS] != NULL) { 174 u64 *pure_flags = nla_data(tb[TCA_SKBEDIT_FLAGS]); 175 176 if (*pure_flags & SKBEDIT_F_TXQ_SKBHASH) { 177 u16 *queue_mapping_max; 178 179 if (!tb[TCA_SKBEDIT_QUEUE_MAPPING] || 180 !tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]) { 181 NL_SET_ERR_MSG_MOD(extack, "Missing required range of queue_mapping."); 182 return -EINVAL; 183 } 184 185 queue_mapping_max = 186 nla_data(tb[TCA_SKBEDIT_QUEUE_MAPPING_MAX]); 187 if (*queue_mapping_max < *queue_mapping) { 188 NL_SET_ERR_MSG_MOD(extack, "The range of queue_mapping is invalid, max < min."); 189 return -EINVAL; 190 } 191 192 mapping_mod = *queue_mapping_max - *queue_mapping + 1; 193 flags |= SKBEDIT_F_TXQ_SKBHASH; 194 } 195 if (*pure_flags & SKBEDIT_F_INHERITDSFIELD) 196 flags |= SKBEDIT_F_INHERITDSFIELD; 197 } 198 199 parm = nla_data(tb[TCA_SKBEDIT_PARMS]); 200 index = parm->index; 201 err = tcf_idr_check_alloc(tn, &index, a, bind); 202 if (err < 0) 203 return err; 204 exists = err; 205 if (exists && bind) 206 return 0; 207 208 if (!flags) { 209 if (exists) 210 tcf_idr_release(*a, bind); 211 else 212 tcf_idr_cleanup(tn, index); 213 return -EINVAL; 214 } 215 216 if (!exists) { 217 ret = tcf_idr_create(tn, index, est, a, 218 &act_skbedit_ops, bind, true, act_flags); 219 if (ret) { 220 tcf_idr_cleanup(tn, index); 221 return ret; 222 } 223 224 d = to_skbedit(*a); 225 ret = ACT_P_CREATED; 226 } else { 227 d = to_skbedit(*a); 228 if (!(act_flags & TCA_ACT_FLAGS_REPLACE)) { 229 tcf_idr_release(*a, bind); 230 return -EEXIST; 231 } 232 } 233 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 234 if (err < 0) 235 goto release_idr; 236 237 params_new = kzalloc(sizeof(*params_new), GFP_KERNEL); 238 if (unlikely(!params_new)) { 239 err = -ENOMEM; 240 goto put_chain; 241 } 242 243 params_new->flags = flags; 244 if (flags & SKBEDIT_F_PRIORITY) 245 params_new->priority = *priority; 246 if (flags & SKBEDIT_F_QUEUE_MAPPING) { 247 params_new->queue_mapping = *queue_mapping; 248 params_new->mapping_mod = mapping_mod; 249 } 250 if (flags & SKBEDIT_F_MARK) 251 params_new->mark = *mark; 252 if (flags & SKBEDIT_F_PTYPE) 253 params_new->ptype = *ptype; 254 /* default behaviour is to use all the bits */ 255 params_new->mask = 0xffffffff; 256 if (flags & SKBEDIT_F_MASK) 257 params_new->mask = *mask; 258 259 spin_lock_bh(&d->tcf_lock); 260 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 261 params_new = rcu_replace_pointer(d->params, params_new, 262 lockdep_is_held(&d->tcf_lock)); 263 spin_unlock_bh(&d->tcf_lock); 264 if (params_new) 265 kfree_rcu(params_new, rcu); 266 if (goto_ch) 267 tcf_chain_put_by_act(goto_ch); 268 269 return ret; 270 put_chain: 271 if (goto_ch) 272 tcf_chain_put_by_act(goto_ch); 273 release_idr: 274 tcf_idr_release(*a, bind); 275 return err; 276 } 277 278 static int tcf_skbedit_dump(struct sk_buff *skb, struct tc_action *a, 279 int bind, int ref) 280 { 281 unsigned char *b = skb_tail_pointer(skb); 282 struct tcf_skbedit *d = to_skbedit(a); 283 struct tcf_skbedit_params *params; 284 struct tc_skbedit opt = { 285 .index = d->tcf_index, 286 .refcnt = refcount_read(&d->tcf_refcnt) - ref, 287 .bindcnt = atomic_read(&d->tcf_bindcnt) - bind, 288 }; 289 u64 pure_flags = 0; 290 struct tcf_t t; 291 292 spin_lock_bh(&d->tcf_lock); 293 params = rcu_dereference_protected(d->params, 294 lockdep_is_held(&d->tcf_lock)); 295 opt.action = d->tcf_action; 296 297 if (nla_put(skb, TCA_SKBEDIT_PARMS, sizeof(opt), &opt)) 298 goto nla_put_failure; 299 if ((params->flags & SKBEDIT_F_PRIORITY) && 300 nla_put_u32(skb, TCA_SKBEDIT_PRIORITY, params->priority)) 301 goto nla_put_failure; 302 if ((params->flags & SKBEDIT_F_QUEUE_MAPPING) && 303 nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING, params->queue_mapping)) 304 goto nla_put_failure; 305 if ((params->flags & SKBEDIT_F_MARK) && 306 nla_put_u32(skb, TCA_SKBEDIT_MARK, params->mark)) 307 goto nla_put_failure; 308 if ((params->flags & SKBEDIT_F_PTYPE) && 309 nla_put_u16(skb, TCA_SKBEDIT_PTYPE, params->ptype)) 310 goto nla_put_failure; 311 if ((params->flags & SKBEDIT_F_MASK) && 312 nla_put_u32(skb, TCA_SKBEDIT_MASK, params->mask)) 313 goto nla_put_failure; 314 if (params->flags & SKBEDIT_F_INHERITDSFIELD) 315 pure_flags |= SKBEDIT_F_INHERITDSFIELD; 316 if (params->flags & SKBEDIT_F_TXQ_SKBHASH) { 317 if (nla_put_u16(skb, TCA_SKBEDIT_QUEUE_MAPPING_MAX, 318 params->queue_mapping + params->mapping_mod - 1)) 319 goto nla_put_failure; 320 321 pure_flags |= SKBEDIT_F_TXQ_SKBHASH; 322 } 323 if (pure_flags != 0 && 324 nla_put(skb, TCA_SKBEDIT_FLAGS, sizeof(pure_flags), &pure_flags)) 325 goto nla_put_failure; 326 327 tcf_tm_dump(&t, &d->tcf_tm); 328 if (nla_put_64bit(skb, TCA_SKBEDIT_TM, sizeof(t), &t, TCA_SKBEDIT_PAD)) 329 goto nla_put_failure; 330 spin_unlock_bh(&d->tcf_lock); 331 332 return skb->len; 333 334 nla_put_failure: 335 spin_unlock_bh(&d->tcf_lock); 336 nlmsg_trim(skb, b); 337 return -1; 338 } 339 340 static void tcf_skbedit_cleanup(struct tc_action *a) 341 { 342 struct tcf_skbedit *d = to_skbedit(a); 343 struct tcf_skbedit_params *params; 344 345 params = rcu_dereference_protected(d->params, 1); 346 if (params) 347 kfree_rcu(params, rcu); 348 } 349 350 static int tcf_skbedit_walker(struct net *net, struct sk_buff *skb, 351 struct netlink_callback *cb, int type, 352 const struct tc_action_ops *ops, 353 struct netlink_ext_ack *extack) 354 { 355 struct tc_action_net *tn = net_generic(net, skbedit_net_id); 356 357 return tcf_generic_walker(tn, skb, cb, type, ops, extack); 358 } 359 360 static int tcf_skbedit_search(struct net *net, struct tc_action **a, u32 index) 361 { 362 struct tc_action_net *tn = net_generic(net, skbedit_net_id); 363 364 return tcf_idr_search(tn, a, index); 365 } 366 367 static size_t tcf_skbedit_get_fill_size(const struct tc_action *act) 368 { 369 return nla_total_size(sizeof(struct tc_skbedit)) 370 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_PRIORITY */ 371 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING */ 372 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_QUEUE_MAPPING_MAX */ 373 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MARK */ 374 + nla_total_size(sizeof(u16)) /* TCA_SKBEDIT_PTYPE */ 375 + nla_total_size(sizeof(u32)) /* TCA_SKBEDIT_MASK */ 376 + nla_total_size_64bit(sizeof(u64)); /* TCA_SKBEDIT_FLAGS */ 377 } 378 379 static int tcf_skbedit_offload_act_setup(struct tc_action *act, void *entry_data, 380 u32 *index_inc, bool bind, 381 struct netlink_ext_ack *extack) 382 { 383 if (bind) { 384 struct flow_action_entry *entry = entry_data; 385 386 if (is_tcf_skbedit_mark(act)) { 387 entry->id = FLOW_ACTION_MARK; 388 entry->mark = tcf_skbedit_mark(act); 389 } else if (is_tcf_skbedit_ptype(act)) { 390 entry->id = FLOW_ACTION_PTYPE; 391 entry->ptype = tcf_skbedit_ptype(act); 392 } else if (is_tcf_skbedit_priority(act)) { 393 entry->id = FLOW_ACTION_PRIORITY; 394 entry->priority = tcf_skbedit_priority(act); 395 } else if (is_tcf_skbedit_queue_mapping(act)) { 396 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"queue_mapping\" option is used"); 397 return -EOPNOTSUPP; 398 } else if (is_tcf_skbedit_inheritdsfield(act)) { 399 NL_SET_ERR_MSG_MOD(extack, "Offload not supported when \"inheritdsfield\" option is used"); 400 return -EOPNOTSUPP; 401 } else { 402 NL_SET_ERR_MSG_MOD(extack, "Unsupported skbedit option offload"); 403 return -EOPNOTSUPP; 404 } 405 *index_inc = 1; 406 } else { 407 struct flow_offload_action *fl_action = entry_data; 408 409 if (is_tcf_skbedit_mark(act)) 410 fl_action->id = FLOW_ACTION_MARK; 411 else if (is_tcf_skbedit_ptype(act)) 412 fl_action->id = FLOW_ACTION_PTYPE; 413 else if (is_tcf_skbedit_priority(act)) 414 fl_action->id = FLOW_ACTION_PRIORITY; 415 else 416 return -EOPNOTSUPP; 417 } 418 419 return 0; 420 } 421 422 static struct tc_action_ops act_skbedit_ops = { 423 .kind = "skbedit", 424 .id = TCA_ID_SKBEDIT, 425 .owner = THIS_MODULE, 426 .act = tcf_skbedit_act, 427 .stats_update = tcf_skbedit_stats_update, 428 .dump = tcf_skbedit_dump, 429 .init = tcf_skbedit_init, 430 .cleanup = tcf_skbedit_cleanup, 431 .walk = tcf_skbedit_walker, 432 .get_fill_size = tcf_skbedit_get_fill_size, 433 .lookup = tcf_skbedit_search, 434 .offload_act_setup = tcf_skbedit_offload_act_setup, 435 .size = sizeof(struct tcf_skbedit), 436 }; 437 438 static __net_init int skbedit_init_net(struct net *net) 439 { 440 struct tc_action_net *tn = net_generic(net, skbedit_net_id); 441 442 return tc_action_net_init(net, tn, &act_skbedit_ops); 443 } 444 445 static void __net_exit skbedit_exit_net(struct list_head *net_list) 446 { 447 tc_action_net_exit(net_list, skbedit_net_id); 448 } 449 450 static struct pernet_operations skbedit_net_ops = { 451 .init = skbedit_init_net, 452 .exit_batch = skbedit_exit_net, 453 .id = &skbedit_net_id, 454 .size = sizeof(struct tc_action_net), 455 }; 456 457 MODULE_AUTHOR("Alexander Duyck, <alexander.h.duyck@intel.com>"); 458 MODULE_DESCRIPTION("SKB Editing"); 459 MODULE_LICENSE("GPL"); 460 461 static int __init skbedit_init_module(void) 462 { 463 return tcf_register_action(&act_skbedit_ops, &skbedit_net_ops); 464 } 465 466 static void __exit skbedit_cleanup_module(void) 467 { 468 tcf_unregister_action(&act_skbedit_ops, &skbedit_net_ops); 469 } 470 471 module_init(skbedit_init_module); 472 module_exit(skbedit_cleanup_module); 473