1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/cls_flower.c Flower classifier 4 * 5 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/init.h> 10 #include <linux/module.h> 11 #include <linux/rhashtable.h> 12 #include <linux/workqueue.h> 13 #include <linux/refcount.h> 14 15 #include <linux/if_ether.h> 16 #include <linux/in6.h> 17 #include <linux/ip.h> 18 #include <linux/mpls.h> 19 20 #include <net/sch_generic.h> 21 #include <net/pkt_cls.h> 22 #include <net/ip.h> 23 #include <net/flow_dissector.h> 24 #include <net/geneve.h> 25 #include <net/vxlan.h> 26 #include <net/erspan.h> 27 28 #include <net/dst.h> 29 #include <net/dst_metadata.h> 30 31 #include <uapi/linux/netfilter/nf_conntrack_common.h> 32 33 struct fl_flow_key { 34 struct flow_dissector_key_meta meta; 35 struct flow_dissector_key_control control; 36 struct flow_dissector_key_control enc_control; 37 struct flow_dissector_key_basic basic; 38 struct flow_dissector_key_eth_addrs eth; 39 struct flow_dissector_key_vlan vlan; 40 struct flow_dissector_key_vlan cvlan; 41 union { 42 struct flow_dissector_key_ipv4_addrs ipv4; 43 struct flow_dissector_key_ipv6_addrs ipv6; 44 }; 45 struct flow_dissector_key_ports tp; 46 struct flow_dissector_key_icmp icmp; 47 struct flow_dissector_key_arp arp; 48 struct flow_dissector_key_keyid enc_key_id; 49 union { 50 struct flow_dissector_key_ipv4_addrs enc_ipv4; 51 struct flow_dissector_key_ipv6_addrs enc_ipv6; 52 }; 53 struct flow_dissector_key_ports enc_tp; 54 struct flow_dissector_key_mpls mpls; 55 struct flow_dissector_key_tcp tcp; 56 struct flow_dissector_key_ip ip; 57 struct flow_dissector_key_ip enc_ip; 58 struct flow_dissector_key_enc_opts enc_opts; 59 union { 60 struct flow_dissector_key_ports tp; 61 struct { 62 struct flow_dissector_key_ports tp_min; 63 struct flow_dissector_key_ports tp_max; 64 }; 65 } tp_range; 66 struct flow_dissector_key_ct ct; 67 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */ 68 69 struct fl_flow_mask_range { 70 unsigned short int start; 71 unsigned short int end; 72 }; 73 74 struct fl_flow_mask { 75 struct fl_flow_key key; 76 struct fl_flow_mask_range range; 77 u32 flags; 78 struct rhash_head ht_node; 79 struct rhashtable ht; 80 struct rhashtable_params filter_ht_params; 81 struct flow_dissector dissector; 82 struct list_head filters; 83 struct rcu_work rwork; 84 struct list_head list; 85 refcount_t refcnt; 86 }; 87 88 struct fl_flow_tmplt { 89 struct fl_flow_key dummy_key; 90 struct fl_flow_key mask; 91 struct flow_dissector dissector; 92 struct tcf_chain *chain; 93 }; 94 95 struct cls_fl_head { 96 struct rhashtable ht; 97 spinlock_t masks_lock; /* Protect masks list */ 98 struct list_head masks; 99 struct list_head hw_filters; 100 struct rcu_work rwork; 101 struct idr handle_idr; 102 }; 103 104 struct cls_fl_filter { 105 struct fl_flow_mask *mask; 106 struct rhash_head ht_node; 107 struct fl_flow_key mkey; 108 struct tcf_exts exts; 109 struct tcf_result res; 110 struct fl_flow_key key; 111 struct list_head list; 112 struct list_head hw_list; 113 u32 handle; 114 u32 flags; 115 u32 in_hw_count; 116 struct rcu_work rwork; 117 struct net_device *hw_dev; 118 /* Flower classifier is unlocked, which means that its reference counter 119 * can be changed concurrently without any kind of external 120 * synchronization. Use atomic reference counter to be concurrency-safe. 121 */ 122 refcount_t refcnt; 123 bool deleted; 124 }; 125 126 static const struct rhashtable_params mask_ht_params = { 127 .key_offset = offsetof(struct fl_flow_mask, key), 128 .key_len = sizeof(struct fl_flow_key), 129 .head_offset = offsetof(struct fl_flow_mask, ht_node), 130 .automatic_shrinking = true, 131 }; 132 133 static unsigned short int fl_mask_range(const struct fl_flow_mask *mask) 134 { 135 return mask->range.end - mask->range.start; 136 } 137 138 static void fl_mask_update_range(struct fl_flow_mask *mask) 139 { 140 const u8 *bytes = (const u8 *) &mask->key; 141 size_t size = sizeof(mask->key); 142 size_t i, first = 0, last; 143 144 for (i = 0; i < size; i++) { 145 if (bytes[i]) { 146 first = i; 147 break; 148 } 149 } 150 last = first; 151 for (i = size - 1; i != first; i--) { 152 if (bytes[i]) { 153 last = i; 154 break; 155 } 156 } 157 mask->range.start = rounddown(first, sizeof(long)); 158 mask->range.end = roundup(last + 1, sizeof(long)); 159 } 160 161 static void *fl_key_get_start(struct fl_flow_key *key, 162 const struct fl_flow_mask *mask) 163 { 164 return (u8 *) key + mask->range.start; 165 } 166 167 static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key, 168 struct fl_flow_mask *mask) 169 { 170 const long *lkey = fl_key_get_start(key, mask); 171 const long *lmask = fl_key_get_start(&mask->key, mask); 172 long *lmkey = fl_key_get_start(mkey, mask); 173 int i; 174 175 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) 176 *lmkey++ = *lkey++ & *lmask++; 177 } 178 179 static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt, 180 struct fl_flow_mask *mask) 181 { 182 const long *lmask = fl_key_get_start(&mask->key, mask); 183 const long *ltmplt; 184 int i; 185 186 if (!tmplt) 187 return true; 188 ltmplt = fl_key_get_start(&tmplt->mask, mask); 189 for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) { 190 if (~*ltmplt++ & *lmask++) 191 return false; 192 } 193 return true; 194 } 195 196 static void fl_clear_masked_range(struct fl_flow_key *key, 197 struct fl_flow_mask *mask) 198 { 199 memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask)); 200 } 201 202 static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter, 203 struct fl_flow_key *key, 204 struct fl_flow_key *mkey) 205 { 206 __be16 min_mask, max_mask, min_val, max_val; 207 208 min_mask = htons(filter->mask->key.tp_range.tp_min.dst); 209 max_mask = htons(filter->mask->key.tp_range.tp_max.dst); 210 min_val = htons(filter->key.tp_range.tp_min.dst); 211 max_val = htons(filter->key.tp_range.tp_max.dst); 212 213 if (min_mask && max_mask) { 214 if (htons(key->tp_range.tp.dst) < min_val || 215 htons(key->tp_range.tp.dst) > max_val) 216 return false; 217 218 /* skb does not have min and max values */ 219 mkey->tp_range.tp_min.dst = filter->mkey.tp_range.tp_min.dst; 220 mkey->tp_range.tp_max.dst = filter->mkey.tp_range.tp_max.dst; 221 } 222 return true; 223 } 224 225 static bool fl_range_port_src_cmp(struct cls_fl_filter *filter, 226 struct fl_flow_key *key, 227 struct fl_flow_key *mkey) 228 { 229 __be16 min_mask, max_mask, min_val, max_val; 230 231 min_mask = htons(filter->mask->key.tp_range.tp_min.src); 232 max_mask = htons(filter->mask->key.tp_range.tp_max.src); 233 min_val = htons(filter->key.tp_range.tp_min.src); 234 max_val = htons(filter->key.tp_range.tp_max.src); 235 236 if (min_mask && max_mask) { 237 if (htons(key->tp_range.tp.src) < min_val || 238 htons(key->tp_range.tp.src) > max_val) 239 return false; 240 241 /* skb does not have min and max values */ 242 mkey->tp_range.tp_min.src = filter->mkey.tp_range.tp_min.src; 243 mkey->tp_range.tp_max.src = filter->mkey.tp_range.tp_max.src; 244 } 245 return true; 246 } 247 248 static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask, 249 struct fl_flow_key *mkey) 250 { 251 return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask), 252 mask->filter_ht_params); 253 } 254 255 static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask, 256 struct fl_flow_key *mkey, 257 struct fl_flow_key *key) 258 { 259 struct cls_fl_filter *filter, *f; 260 261 list_for_each_entry_rcu(filter, &mask->filters, list) { 262 if (!fl_range_port_dst_cmp(filter, key, mkey)) 263 continue; 264 265 if (!fl_range_port_src_cmp(filter, key, mkey)) 266 continue; 267 268 f = __fl_lookup(mask, mkey); 269 if (f) 270 return f; 271 } 272 return NULL; 273 } 274 275 static noinline_for_stack 276 struct cls_fl_filter *fl_mask_lookup(struct fl_flow_mask *mask, struct fl_flow_key *key) 277 { 278 struct fl_flow_key mkey; 279 280 fl_set_masked_key(&mkey, key, mask); 281 if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE)) 282 return fl_lookup_range(mask, &mkey, key); 283 284 return __fl_lookup(mask, &mkey); 285 } 286 287 static u16 fl_ct_info_to_flower_map[] = { 288 [IP_CT_ESTABLISHED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 289 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED, 290 [IP_CT_RELATED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 291 TCA_FLOWER_KEY_CT_FLAGS_RELATED, 292 [IP_CT_ESTABLISHED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 293 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED, 294 [IP_CT_RELATED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 295 TCA_FLOWER_KEY_CT_FLAGS_RELATED, 296 [IP_CT_NEW] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED | 297 TCA_FLOWER_KEY_CT_FLAGS_NEW, 298 }; 299 300 static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp, 301 struct tcf_result *res) 302 { 303 struct cls_fl_head *head = rcu_dereference_bh(tp->root); 304 struct fl_flow_key skb_key; 305 struct fl_flow_mask *mask; 306 struct cls_fl_filter *f; 307 308 list_for_each_entry_rcu(mask, &head->masks, list) { 309 flow_dissector_init_keys(&skb_key.control, &skb_key.basic); 310 fl_clear_masked_range(&skb_key, mask); 311 312 skb_flow_dissect_meta(skb, &mask->dissector, &skb_key); 313 /* skb_flow_dissect() does not set n_proto in case an unknown 314 * protocol, so do it rather here. 315 */ 316 skb_key.basic.n_proto = skb->protocol; 317 skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key); 318 skb_flow_dissect_ct(skb, &mask->dissector, &skb_key, 319 fl_ct_info_to_flower_map, 320 ARRAY_SIZE(fl_ct_info_to_flower_map)); 321 skb_flow_dissect(skb, &mask->dissector, &skb_key, 0); 322 323 f = fl_mask_lookup(mask, &skb_key); 324 if (f && !tc_skip_sw(f->flags)) { 325 *res = f->res; 326 return tcf_exts_exec(skb, &f->exts, res); 327 } 328 } 329 return -1; 330 } 331 332 static int fl_init(struct tcf_proto *tp) 333 { 334 struct cls_fl_head *head; 335 336 head = kzalloc(sizeof(*head), GFP_KERNEL); 337 if (!head) 338 return -ENOBUFS; 339 340 spin_lock_init(&head->masks_lock); 341 INIT_LIST_HEAD_RCU(&head->masks); 342 INIT_LIST_HEAD(&head->hw_filters); 343 rcu_assign_pointer(tp->root, head); 344 idr_init(&head->handle_idr); 345 346 return rhashtable_init(&head->ht, &mask_ht_params); 347 } 348 349 static void fl_mask_free(struct fl_flow_mask *mask, bool mask_init_done) 350 { 351 /* temporary masks don't have their filters list and ht initialized */ 352 if (mask_init_done) { 353 WARN_ON(!list_empty(&mask->filters)); 354 rhashtable_destroy(&mask->ht); 355 } 356 kfree(mask); 357 } 358 359 static void fl_mask_free_work(struct work_struct *work) 360 { 361 struct fl_flow_mask *mask = container_of(to_rcu_work(work), 362 struct fl_flow_mask, rwork); 363 364 fl_mask_free(mask, true); 365 } 366 367 static void fl_uninit_mask_free_work(struct work_struct *work) 368 { 369 struct fl_flow_mask *mask = container_of(to_rcu_work(work), 370 struct fl_flow_mask, rwork); 371 372 fl_mask_free(mask, false); 373 } 374 375 static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask) 376 { 377 if (!refcount_dec_and_test(&mask->refcnt)) 378 return false; 379 380 rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params); 381 382 spin_lock(&head->masks_lock); 383 list_del_rcu(&mask->list); 384 spin_unlock(&head->masks_lock); 385 386 tcf_queue_work(&mask->rwork, fl_mask_free_work); 387 388 return true; 389 } 390 391 static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp) 392 { 393 /* Flower classifier only changes root pointer during init and destroy. 394 * Users must obtain reference to tcf_proto instance before calling its 395 * API, so tp->root pointer is protected from concurrent call to 396 * fl_destroy() by reference counting. 397 */ 398 return rcu_dereference_raw(tp->root); 399 } 400 401 static void __fl_destroy_filter(struct cls_fl_filter *f) 402 { 403 tcf_exts_destroy(&f->exts); 404 tcf_exts_put_net(&f->exts); 405 kfree(f); 406 } 407 408 static void fl_destroy_filter_work(struct work_struct *work) 409 { 410 struct cls_fl_filter *f = container_of(to_rcu_work(work), 411 struct cls_fl_filter, rwork); 412 413 __fl_destroy_filter(f); 414 } 415 416 static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f, 417 bool rtnl_held, struct netlink_ext_ack *extack) 418 { 419 struct tcf_block *block = tp->chain->block; 420 struct flow_cls_offload cls_flower = {}; 421 422 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 423 cls_flower.command = FLOW_CLS_DESTROY; 424 cls_flower.cookie = (unsigned long) f; 425 426 tc_setup_cb_destroy(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, false, 427 &f->flags, &f->in_hw_count, rtnl_held); 428 429 } 430 431 static int fl_hw_replace_filter(struct tcf_proto *tp, 432 struct cls_fl_filter *f, bool rtnl_held, 433 struct netlink_ext_ack *extack) 434 { 435 struct tcf_block *block = tp->chain->block; 436 struct flow_cls_offload cls_flower = {}; 437 bool skip_sw = tc_skip_sw(f->flags); 438 int err = 0; 439 440 cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 441 if (!cls_flower.rule) 442 return -ENOMEM; 443 444 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack); 445 cls_flower.command = FLOW_CLS_REPLACE; 446 cls_flower.cookie = (unsigned long) f; 447 cls_flower.rule->match.dissector = &f->mask->dissector; 448 cls_flower.rule->match.mask = &f->mask->key; 449 cls_flower.rule->match.key = &f->mkey; 450 cls_flower.classid = f->res.classid; 451 452 err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 453 if (err) { 454 kfree(cls_flower.rule); 455 if (skip_sw) { 456 NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 457 return err; 458 } 459 return 0; 460 } 461 462 err = tc_setup_cb_add(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, 463 skip_sw, &f->flags, &f->in_hw_count, rtnl_held); 464 tc_cleanup_flow_action(&cls_flower.rule->action); 465 kfree(cls_flower.rule); 466 467 if (err) { 468 fl_hw_destroy_filter(tp, f, rtnl_held, NULL); 469 return err; 470 } 471 472 if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW)) 473 return -EINVAL; 474 475 return 0; 476 } 477 478 static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f, 479 bool rtnl_held) 480 { 481 struct tcf_block *block = tp->chain->block; 482 struct flow_cls_offload cls_flower = {}; 483 484 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL); 485 cls_flower.command = FLOW_CLS_STATS; 486 cls_flower.cookie = (unsigned long) f; 487 cls_flower.classid = f->res.classid; 488 489 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, 490 rtnl_held); 491 492 tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes, 493 cls_flower.stats.pkts, 494 cls_flower.stats.lastused, 495 cls_flower.stats.used_hw_stats, 496 cls_flower.stats.used_hw_stats_valid); 497 } 498 499 static void __fl_put(struct cls_fl_filter *f) 500 { 501 if (!refcount_dec_and_test(&f->refcnt)) 502 return; 503 504 if (tcf_exts_get_net(&f->exts)) 505 tcf_queue_work(&f->rwork, fl_destroy_filter_work); 506 else 507 __fl_destroy_filter(f); 508 } 509 510 static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle) 511 { 512 struct cls_fl_filter *f; 513 514 rcu_read_lock(); 515 f = idr_find(&head->handle_idr, handle); 516 if (f && !refcount_inc_not_zero(&f->refcnt)) 517 f = NULL; 518 rcu_read_unlock(); 519 520 return f; 521 } 522 523 static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f, 524 bool *last, bool rtnl_held, 525 struct netlink_ext_ack *extack) 526 { 527 struct cls_fl_head *head = fl_head_dereference(tp); 528 529 *last = false; 530 531 spin_lock(&tp->lock); 532 if (f->deleted) { 533 spin_unlock(&tp->lock); 534 return -ENOENT; 535 } 536 537 f->deleted = true; 538 rhashtable_remove_fast(&f->mask->ht, &f->ht_node, 539 f->mask->filter_ht_params); 540 idr_remove(&head->handle_idr, f->handle); 541 list_del_rcu(&f->list); 542 spin_unlock(&tp->lock); 543 544 *last = fl_mask_put(head, f->mask); 545 if (!tc_skip_hw(f->flags)) 546 fl_hw_destroy_filter(tp, f, rtnl_held, extack); 547 tcf_unbind_filter(tp, &f->res); 548 __fl_put(f); 549 550 return 0; 551 } 552 553 static void fl_destroy_sleepable(struct work_struct *work) 554 { 555 struct cls_fl_head *head = container_of(to_rcu_work(work), 556 struct cls_fl_head, 557 rwork); 558 559 rhashtable_destroy(&head->ht); 560 kfree(head); 561 module_put(THIS_MODULE); 562 } 563 564 static void fl_destroy(struct tcf_proto *tp, bool rtnl_held, 565 struct netlink_ext_ack *extack) 566 { 567 struct cls_fl_head *head = fl_head_dereference(tp); 568 struct fl_flow_mask *mask, *next_mask; 569 struct cls_fl_filter *f, *next; 570 bool last; 571 572 list_for_each_entry_safe(mask, next_mask, &head->masks, list) { 573 list_for_each_entry_safe(f, next, &mask->filters, list) { 574 __fl_delete(tp, f, &last, rtnl_held, extack); 575 if (last) 576 break; 577 } 578 } 579 idr_destroy(&head->handle_idr); 580 581 __module_get(THIS_MODULE); 582 tcf_queue_work(&head->rwork, fl_destroy_sleepable); 583 } 584 585 static void fl_put(struct tcf_proto *tp, void *arg) 586 { 587 struct cls_fl_filter *f = arg; 588 589 __fl_put(f); 590 } 591 592 static void *fl_get(struct tcf_proto *tp, u32 handle) 593 { 594 struct cls_fl_head *head = fl_head_dereference(tp); 595 596 return __fl_get(head, handle); 597 } 598 599 static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = { 600 [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC }, 601 [TCA_FLOWER_CLASSID] = { .type = NLA_U32 }, 602 [TCA_FLOWER_INDEV] = { .type = NLA_STRING, 603 .len = IFNAMSIZ }, 604 [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN }, 605 [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN }, 606 [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN }, 607 [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN }, 608 [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 }, 609 [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 }, 610 [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 }, 611 [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 }, 612 [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 }, 613 [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 }, 614 [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 615 [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 616 [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 617 [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 618 [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 }, 619 [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 }, 620 [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 }, 621 [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 }, 622 [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 }, 623 [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 }, 624 [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 }, 625 [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 }, 626 [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 }, 627 [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 }, 628 [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 }, 629 [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 }, 630 [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) }, 631 [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) }, 632 [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) }, 633 [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) }, 634 [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 }, 635 [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 }, 636 [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 }, 637 [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 }, 638 [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 }, 639 [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 }, 640 [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 }, 641 [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 }, 642 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 }, 643 [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 }, 644 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 }, 645 [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 }, 646 [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 }, 647 [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 }, 648 [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 }, 649 [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 }, 650 [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 }, 651 [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 }, 652 [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 }, 653 [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 }, 654 [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 }, 655 [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 }, 656 [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 }, 657 [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 }, 658 [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 }, 659 [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 }, 660 [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 }, 661 [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 }, 662 [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN }, 663 [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN }, 664 [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN }, 665 [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN }, 666 [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 }, 667 [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 }, 668 [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 }, 669 [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 }, 670 [TCA_FLOWER_KEY_MPLS_OPTS] = { .type = NLA_NESTED }, 671 [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 }, 672 [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 }, 673 [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 }, 674 [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 }, 675 [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 }, 676 [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 }, 677 [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 }, 678 [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 }, 679 [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 }, 680 [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 }, 681 [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 }, 682 [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 }, 683 [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 }, 684 [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED }, 685 [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED }, 686 [TCA_FLOWER_KEY_CT_STATE] = { .type = NLA_U16 }, 687 [TCA_FLOWER_KEY_CT_STATE_MASK] = { .type = NLA_U16 }, 688 [TCA_FLOWER_KEY_CT_ZONE] = { .type = NLA_U16 }, 689 [TCA_FLOWER_KEY_CT_ZONE_MASK] = { .type = NLA_U16 }, 690 [TCA_FLOWER_KEY_CT_MARK] = { .type = NLA_U32 }, 691 [TCA_FLOWER_KEY_CT_MARK_MASK] = { .type = NLA_U32 }, 692 [TCA_FLOWER_KEY_CT_LABELS] = { .type = NLA_BINARY, 693 .len = 128 / BITS_PER_BYTE }, 694 [TCA_FLOWER_KEY_CT_LABELS_MASK] = { .type = NLA_BINARY, 695 .len = 128 / BITS_PER_BYTE }, 696 [TCA_FLOWER_FLAGS] = { .type = NLA_U32 }, 697 }; 698 699 static const struct nla_policy 700 enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = { 701 [TCA_FLOWER_KEY_ENC_OPTS_UNSPEC] = { 702 .strict_start_type = TCA_FLOWER_KEY_ENC_OPTS_VXLAN }, 703 [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED }, 704 [TCA_FLOWER_KEY_ENC_OPTS_VXLAN] = { .type = NLA_NESTED }, 705 [TCA_FLOWER_KEY_ENC_OPTS_ERSPAN] = { .type = NLA_NESTED }, 706 }; 707 708 static const struct nla_policy 709 geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = { 710 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 }, 711 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 }, 712 [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY, 713 .len = 128 }, 714 }; 715 716 static const struct nla_policy 717 vxlan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1] = { 718 [TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP] = { .type = NLA_U32 }, 719 }; 720 721 static const struct nla_policy 722 erspan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1] = { 723 [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER] = { .type = NLA_U8 }, 724 [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX] = { .type = NLA_U32 }, 725 [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] = { .type = NLA_U8 }, 726 [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID] = { .type = NLA_U8 }, 727 }; 728 729 static const struct nla_policy 730 mpls_stack_entry_policy[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1] = { 731 [TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH] = { .type = NLA_U8 }, 732 [TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL] = { .type = NLA_U8 }, 733 [TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS] = { .type = NLA_U8 }, 734 [TCA_FLOWER_KEY_MPLS_OPT_LSE_TC] = { .type = NLA_U8 }, 735 [TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL] = { .type = NLA_U32 }, 736 }; 737 738 static void fl_set_key_val(struct nlattr **tb, 739 void *val, int val_type, 740 void *mask, int mask_type, int len) 741 { 742 if (!tb[val_type]) 743 return; 744 nla_memcpy(val, tb[val_type], len); 745 if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type]) 746 memset(mask, 0xff, len); 747 else 748 nla_memcpy(mask, tb[mask_type], len); 749 } 750 751 static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key, 752 struct fl_flow_key *mask, 753 struct netlink_ext_ack *extack) 754 { 755 fl_set_key_val(tb, &key->tp_range.tp_min.dst, 756 TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_range.tp_min.dst, 757 TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.dst)); 758 fl_set_key_val(tb, &key->tp_range.tp_max.dst, 759 TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_range.tp_max.dst, 760 TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.dst)); 761 fl_set_key_val(tb, &key->tp_range.tp_min.src, 762 TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_range.tp_min.src, 763 TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.src)); 764 fl_set_key_val(tb, &key->tp_range.tp_max.src, 765 TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_range.tp_max.src, 766 TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.src)); 767 768 if (mask->tp_range.tp_min.dst && mask->tp_range.tp_max.dst && 769 htons(key->tp_range.tp_max.dst) <= 770 htons(key->tp_range.tp_min.dst)) { 771 NL_SET_ERR_MSG_ATTR(extack, 772 tb[TCA_FLOWER_KEY_PORT_DST_MIN], 773 "Invalid destination port range (min must be strictly smaller than max)"); 774 return -EINVAL; 775 } 776 if (mask->tp_range.tp_min.src && mask->tp_range.tp_max.src && 777 htons(key->tp_range.tp_max.src) <= 778 htons(key->tp_range.tp_min.src)) { 779 NL_SET_ERR_MSG_ATTR(extack, 780 tb[TCA_FLOWER_KEY_PORT_SRC_MIN], 781 "Invalid source port range (min must be strictly smaller than max)"); 782 return -EINVAL; 783 } 784 785 return 0; 786 } 787 788 static int fl_set_key_mpls_lse(const struct nlattr *nla_lse, 789 struct flow_dissector_key_mpls *key_val, 790 struct flow_dissector_key_mpls *key_mask, 791 struct netlink_ext_ack *extack) 792 { 793 struct nlattr *tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1]; 794 struct flow_dissector_mpls_lse *lse_mask; 795 struct flow_dissector_mpls_lse *lse_val; 796 u8 lse_index; 797 u8 depth; 798 int err; 799 800 err = nla_parse_nested(tb, TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX, nla_lse, 801 mpls_stack_entry_policy, extack); 802 if (err < 0) 803 return err; 804 805 if (!tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]) { 806 NL_SET_ERR_MSG(extack, "Missing MPLS option \"depth\""); 807 return -EINVAL; 808 } 809 810 depth = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]); 811 812 /* LSE depth starts at 1, for consistency with terminology used by 813 * RFC 3031 (section 3.9), where depth 0 refers to unlabeled packets. 814 */ 815 if (depth < 1 || depth > FLOW_DIS_MPLS_MAX) { 816 NL_SET_ERR_MSG_ATTR(extack, 817 tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH], 818 "Invalid MPLS depth"); 819 return -EINVAL; 820 } 821 lse_index = depth - 1; 822 823 dissector_set_mpls_lse(key_val, lse_index); 824 dissector_set_mpls_lse(key_mask, lse_index); 825 826 lse_val = &key_val->ls[lse_index]; 827 lse_mask = &key_mask->ls[lse_index]; 828 829 if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]) { 830 lse_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]); 831 lse_mask->mpls_ttl = MPLS_TTL_MASK; 832 } 833 if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]) { 834 u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]); 835 836 if (bos & ~MPLS_BOS_MASK) { 837 NL_SET_ERR_MSG_ATTR(extack, 838 tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS], 839 "Bottom Of Stack (BOS) must be 0 or 1"); 840 return -EINVAL; 841 } 842 lse_val->mpls_bos = bos; 843 lse_mask->mpls_bos = MPLS_BOS_MASK; 844 } 845 if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]) { 846 u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]); 847 848 if (tc & ~MPLS_TC_MASK) { 849 NL_SET_ERR_MSG_ATTR(extack, 850 tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC], 851 "Traffic Class (TC) must be between 0 and 7"); 852 return -EINVAL; 853 } 854 lse_val->mpls_tc = tc; 855 lse_mask->mpls_tc = MPLS_TC_MASK; 856 } 857 if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]) { 858 u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]); 859 860 if (label & ~MPLS_LABEL_MASK) { 861 NL_SET_ERR_MSG_ATTR(extack, 862 tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL], 863 "Label must be between 0 and 1048575"); 864 return -EINVAL; 865 } 866 lse_val->mpls_label = label; 867 lse_mask->mpls_label = MPLS_LABEL_MASK; 868 } 869 870 return 0; 871 } 872 873 static int fl_set_key_mpls_opts(const struct nlattr *nla_mpls_opts, 874 struct flow_dissector_key_mpls *key_val, 875 struct flow_dissector_key_mpls *key_mask, 876 struct netlink_ext_ack *extack) 877 { 878 struct nlattr *nla_lse; 879 int rem; 880 int err; 881 882 if (!(nla_mpls_opts->nla_type & NLA_F_NESTED)) { 883 NL_SET_ERR_MSG_ATTR(extack, nla_mpls_opts, 884 "NLA_F_NESTED is missing"); 885 return -EINVAL; 886 } 887 888 nla_for_each_nested(nla_lse, nla_mpls_opts, rem) { 889 if (nla_type(nla_lse) != TCA_FLOWER_KEY_MPLS_OPTS_LSE) { 890 NL_SET_ERR_MSG_ATTR(extack, nla_lse, 891 "Invalid MPLS option type"); 892 return -EINVAL; 893 } 894 895 err = fl_set_key_mpls_lse(nla_lse, key_val, key_mask, extack); 896 if (err < 0) 897 return err; 898 } 899 if (rem) { 900 NL_SET_ERR_MSG(extack, 901 "Bytes leftover after parsing MPLS options"); 902 return -EINVAL; 903 } 904 905 return 0; 906 } 907 908 static int fl_set_key_mpls(struct nlattr **tb, 909 struct flow_dissector_key_mpls *key_val, 910 struct flow_dissector_key_mpls *key_mask, 911 struct netlink_ext_ack *extack) 912 { 913 struct flow_dissector_mpls_lse *lse_mask; 914 struct flow_dissector_mpls_lse *lse_val; 915 916 if (tb[TCA_FLOWER_KEY_MPLS_OPTS]) { 917 if (tb[TCA_FLOWER_KEY_MPLS_TTL] || 918 tb[TCA_FLOWER_KEY_MPLS_BOS] || 919 tb[TCA_FLOWER_KEY_MPLS_TC] || 920 tb[TCA_FLOWER_KEY_MPLS_LABEL]) { 921 NL_SET_ERR_MSG_ATTR(extack, 922 tb[TCA_FLOWER_KEY_MPLS_OPTS], 923 "MPLS label, Traffic Class, Bottom Of Stack and Time To Live must be encapsulated in the MPLS options attribute"); 924 return -EBADMSG; 925 } 926 927 return fl_set_key_mpls_opts(tb[TCA_FLOWER_KEY_MPLS_OPTS], 928 key_val, key_mask, extack); 929 } 930 931 lse_val = &key_val->ls[0]; 932 lse_mask = &key_mask->ls[0]; 933 934 if (tb[TCA_FLOWER_KEY_MPLS_TTL]) { 935 lse_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]); 936 lse_mask->mpls_ttl = MPLS_TTL_MASK; 937 dissector_set_mpls_lse(key_val, 0); 938 dissector_set_mpls_lse(key_mask, 0); 939 } 940 if (tb[TCA_FLOWER_KEY_MPLS_BOS]) { 941 u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]); 942 943 if (bos & ~MPLS_BOS_MASK) { 944 NL_SET_ERR_MSG_ATTR(extack, 945 tb[TCA_FLOWER_KEY_MPLS_BOS], 946 "Bottom Of Stack (BOS) must be 0 or 1"); 947 return -EINVAL; 948 } 949 lse_val->mpls_bos = bos; 950 lse_mask->mpls_bos = MPLS_BOS_MASK; 951 dissector_set_mpls_lse(key_val, 0); 952 dissector_set_mpls_lse(key_mask, 0); 953 } 954 if (tb[TCA_FLOWER_KEY_MPLS_TC]) { 955 u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]); 956 957 if (tc & ~MPLS_TC_MASK) { 958 NL_SET_ERR_MSG_ATTR(extack, 959 tb[TCA_FLOWER_KEY_MPLS_TC], 960 "Traffic Class (TC) must be between 0 and 7"); 961 return -EINVAL; 962 } 963 lse_val->mpls_tc = tc; 964 lse_mask->mpls_tc = MPLS_TC_MASK; 965 dissector_set_mpls_lse(key_val, 0); 966 dissector_set_mpls_lse(key_mask, 0); 967 } 968 if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) { 969 u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]); 970 971 if (label & ~MPLS_LABEL_MASK) { 972 NL_SET_ERR_MSG_ATTR(extack, 973 tb[TCA_FLOWER_KEY_MPLS_LABEL], 974 "Label must be between 0 and 1048575"); 975 return -EINVAL; 976 } 977 lse_val->mpls_label = label; 978 lse_mask->mpls_label = MPLS_LABEL_MASK; 979 dissector_set_mpls_lse(key_val, 0); 980 dissector_set_mpls_lse(key_mask, 0); 981 } 982 return 0; 983 } 984 985 static void fl_set_key_vlan(struct nlattr **tb, 986 __be16 ethertype, 987 int vlan_id_key, int vlan_prio_key, 988 struct flow_dissector_key_vlan *key_val, 989 struct flow_dissector_key_vlan *key_mask) 990 { 991 #define VLAN_PRIORITY_MASK 0x7 992 993 if (tb[vlan_id_key]) { 994 key_val->vlan_id = 995 nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK; 996 key_mask->vlan_id = VLAN_VID_MASK; 997 } 998 if (tb[vlan_prio_key]) { 999 key_val->vlan_priority = 1000 nla_get_u8(tb[vlan_prio_key]) & 1001 VLAN_PRIORITY_MASK; 1002 key_mask->vlan_priority = VLAN_PRIORITY_MASK; 1003 } 1004 key_val->vlan_tpid = ethertype; 1005 key_mask->vlan_tpid = cpu_to_be16(~0); 1006 } 1007 1008 static void fl_set_key_flag(u32 flower_key, u32 flower_mask, 1009 u32 *dissector_key, u32 *dissector_mask, 1010 u32 flower_flag_bit, u32 dissector_flag_bit) 1011 { 1012 if (flower_mask & flower_flag_bit) { 1013 *dissector_mask |= dissector_flag_bit; 1014 if (flower_key & flower_flag_bit) 1015 *dissector_key |= dissector_flag_bit; 1016 } 1017 } 1018 1019 static int fl_set_key_flags(struct nlattr **tb, u32 *flags_key, 1020 u32 *flags_mask, struct netlink_ext_ack *extack) 1021 { 1022 u32 key, mask; 1023 1024 /* mask is mandatory for flags */ 1025 if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) { 1026 NL_SET_ERR_MSG(extack, "Missing flags mask"); 1027 return -EINVAL; 1028 } 1029 1030 key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS])); 1031 mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK])); 1032 1033 *flags_key = 0; 1034 *flags_mask = 0; 1035 1036 fl_set_key_flag(key, mask, flags_key, flags_mask, 1037 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 1038 fl_set_key_flag(key, mask, flags_key, flags_mask, 1039 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 1040 FLOW_DIS_FIRST_FRAG); 1041 1042 return 0; 1043 } 1044 1045 static void fl_set_key_ip(struct nlattr **tb, bool encap, 1046 struct flow_dissector_key_ip *key, 1047 struct flow_dissector_key_ip *mask) 1048 { 1049 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 1050 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 1051 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 1052 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 1053 1054 fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)); 1055 fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)); 1056 } 1057 1058 static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key, 1059 int depth, int option_len, 1060 struct netlink_ext_ack *extack) 1061 { 1062 struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1]; 1063 struct nlattr *class = NULL, *type = NULL, *data = NULL; 1064 struct geneve_opt *opt; 1065 int err, data_len = 0; 1066 1067 if (option_len > sizeof(struct geneve_opt)) 1068 data_len = option_len - sizeof(struct geneve_opt); 1069 1070 opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len]; 1071 memset(opt, 0xff, option_len); 1072 opt->length = data_len / 4; 1073 opt->r1 = 0; 1074 opt->r2 = 0; 1075 opt->r3 = 0; 1076 1077 /* If no mask has been prodived we assume an exact match. */ 1078 if (!depth) 1079 return sizeof(struct geneve_opt) + data_len; 1080 1081 if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) { 1082 NL_SET_ERR_MSG(extack, "Non-geneve option type for mask"); 1083 return -EINVAL; 1084 } 1085 1086 err = nla_parse_nested_deprecated(tb, 1087 TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX, 1088 nla, geneve_opt_policy, extack); 1089 if (err < 0) 1090 return err; 1091 1092 /* We are not allowed to omit any of CLASS, TYPE or DATA 1093 * fields from the key. 1094 */ 1095 if (!option_len && 1096 (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] || 1097 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] || 1098 !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) { 1099 NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data"); 1100 return -EINVAL; 1101 } 1102 1103 /* Omitting any of CLASS, TYPE or DATA fields is allowed 1104 * for the mask. 1105 */ 1106 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) { 1107 int new_len = key->enc_opts.len; 1108 1109 data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]; 1110 data_len = nla_len(data); 1111 if (data_len < 4) { 1112 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long"); 1113 return -ERANGE; 1114 } 1115 if (data_len % 4) { 1116 NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long"); 1117 return -ERANGE; 1118 } 1119 1120 new_len += sizeof(struct geneve_opt) + data_len; 1121 BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX); 1122 if (new_len > FLOW_DIS_TUN_OPTS_MAX) { 1123 NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size"); 1124 return -ERANGE; 1125 } 1126 opt->length = data_len / 4; 1127 memcpy(opt->opt_data, nla_data(data), data_len); 1128 } 1129 1130 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) { 1131 class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]; 1132 opt->opt_class = nla_get_be16(class); 1133 } 1134 1135 if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) { 1136 type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]; 1137 opt->type = nla_get_u8(type); 1138 } 1139 1140 return sizeof(struct geneve_opt) + data_len; 1141 } 1142 1143 static int fl_set_vxlan_opt(const struct nlattr *nla, struct fl_flow_key *key, 1144 int depth, int option_len, 1145 struct netlink_ext_ack *extack) 1146 { 1147 struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1]; 1148 struct vxlan_metadata *md; 1149 int err; 1150 1151 md = (struct vxlan_metadata *)&key->enc_opts.data[key->enc_opts.len]; 1152 memset(md, 0xff, sizeof(*md)); 1153 1154 if (!depth) 1155 return sizeof(*md); 1156 1157 if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_VXLAN) { 1158 NL_SET_ERR_MSG(extack, "Non-vxlan option type for mask"); 1159 return -EINVAL; 1160 } 1161 1162 err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX, nla, 1163 vxlan_opt_policy, extack); 1164 if (err < 0) 1165 return err; 1166 1167 if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) { 1168 NL_SET_ERR_MSG(extack, "Missing tunnel key vxlan option gbp"); 1169 return -EINVAL; 1170 } 1171 1172 if (tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) 1173 md->gbp = nla_get_u32(tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]); 1174 1175 return sizeof(*md); 1176 } 1177 1178 static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key, 1179 int depth, int option_len, 1180 struct netlink_ext_ack *extack) 1181 { 1182 struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1]; 1183 struct erspan_metadata *md; 1184 int err; 1185 1186 md = (struct erspan_metadata *)&key->enc_opts.data[key->enc_opts.len]; 1187 memset(md, 0xff, sizeof(*md)); 1188 md->version = 1; 1189 1190 if (!depth) 1191 return sizeof(*md); 1192 1193 if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_ERSPAN) { 1194 NL_SET_ERR_MSG(extack, "Non-erspan option type for mask"); 1195 return -EINVAL; 1196 } 1197 1198 err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX, nla, 1199 erspan_opt_policy, extack); 1200 if (err < 0) 1201 return err; 1202 1203 if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) { 1204 NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option ver"); 1205 return -EINVAL; 1206 } 1207 1208 if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) 1209 md->version = nla_get_u8(tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]); 1210 1211 if (md->version == 1) { 1212 if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) { 1213 NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option index"); 1214 return -EINVAL; 1215 } 1216 if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) { 1217 nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]; 1218 md->u.index = nla_get_be32(nla); 1219 } 1220 } else if (md->version == 2) { 1221 if (!option_len && (!tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] || 1222 !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID])) { 1223 NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option dir or hwid"); 1224 return -EINVAL; 1225 } 1226 if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]) { 1227 nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]; 1228 md->u.md2.dir = nla_get_u8(nla); 1229 } 1230 if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]) { 1231 nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]; 1232 set_hwid(&md->u.md2, nla_get_u8(nla)); 1233 } 1234 } else { 1235 NL_SET_ERR_MSG(extack, "Tunnel key erspan option ver is incorrect"); 1236 return -EINVAL; 1237 } 1238 1239 return sizeof(*md); 1240 } 1241 1242 static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key, 1243 struct fl_flow_key *mask, 1244 struct netlink_ext_ack *extack) 1245 { 1246 const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL; 1247 int err, option_len, key_depth, msk_depth = 0; 1248 1249 err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS], 1250 TCA_FLOWER_KEY_ENC_OPTS_MAX, 1251 enc_opts_policy, extack); 1252 if (err) 1253 return err; 1254 1255 nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]); 1256 1257 if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) { 1258 err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK], 1259 TCA_FLOWER_KEY_ENC_OPTS_MAX, 1260 enc_opts_policy, extack); 1261 if (err) 1262 return err; 1263 1264 nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 1265 msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]); 1266 } 1267 1268 nla_for_each_attr(nla_opt_key, nla_enc_key, 1269 nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) { 1270 switch (nla_type(nla_opt_key)) { 1271 case TCA_FLOWER_KEY_ENC_OPTS_GENEVE: 1272 if (key->enc_opts.dst_opt_type && 1273 key->enc_opts.dst_opt_type != TUNNEL_GENEVE_OPT) { 1274 NL_SET_ERR_MSG(extack, "Duplicate type for geneve options"); 1275 return -EINVAL; 1276 } 1277 option_len = 0; 1278 key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 1279 option_len = fl_set_geneve_opt(nla_opt_key, key, 1280 key_depth, option_len, 1281 extack); 1282 if (option_len < 0) 1283 return option_len; 1284 1285 key->enc_opts.len += option_len; 1286 /* At the same time we need to parse through the mask 1287 * in order to verify exact and mask attribute lengths. 1288 */ 1289 mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT; 1290 option_len = fl_set_geneve_opt(nla_opt_msk, mask, 1291 msk_depth, option_len, 1292 extack); 1293 if (option_len < 0) 1294 return option_len; 1295 1296 mask->enc_opts.len += option_len; 1297 if (key->enc_opts.len != mask->enc_opts.len) { 1298 NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 1299 return -EINVAL; 1300 } 1301 1302 if (msk_depth) 1303 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 1304 break; 1305 case TCA_FLOWER_KEY_ENC_OPTS_VXLAN: 1306 if (key->enc_opts.dst_opt_type) { 1307 NL_SET_ERR_MSG(extack, "Duplicate type for vxlan options"); 1308 return -EINVAL; 1309 } 1310 option_len = 0; 1311 key->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT; 1312 option_len = fl_set_vxlan_opt(nla_opt_key, key, 1313 key_depth, option_len, 1314 extack); 1315 if (option_len < 0) 1316 return option_len; 1317 1318 key->enc_opts.len += option_len; 1319 /* At the same time we need to parse through the mask 1320 * in order to verify exact and mask attribute lengths. 1321 */ 1322 mask->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT; 1323 option_len = fl_set_vxlan_opt(nla_opt_msk, mask, 1324 msk_depth, option_len, 1325 extack); 1326 if (option_len < 0) 1327 return option_len; 1328 1329 mask->enc_opts.len += option_len; 1330 if (key->enc_opts.len != mask->enc_opts.len) { 1331 NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 1332 return -EINVAL; 1333 } 1334 1335 if (msk_depth) 1336 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 1337 break; 1338 case TCA_FLOWER_KEY_ENC_OPTS_ERSPAN: 1339 if (key->enc_opts.dst_opt_type) { 1340 NL_SET_ERR_MSG(extack, "Duplicate type for erspan options"); 1341 return -EINVAL; 1342 } 1343 option_len = 0; 1344 key->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT; 1345 option_len = fl_set_erspan_opt(nla_opt_key, key, 1346 key_depth, option_len, 1347 extack); 1348 if (option_len < 0) 1349 return option_len; 1350 1351 key->enc_opts.len += option_len; 1352 /* At the same time we need to parse through the mask 1353 * in order to verify exact and mask attribute lengths. 1354 */ 1355 mask->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT; 1356 option_len = fl_set_erspan_opt(nla_opt_msk, mask, 1357 msk_depth, option_len, 1358 extack); 1359 if (option_len < 0) 1360 return option_len; 1361 1362 mask->enc_opts.len += option_len; 1363 if (key->enc_opts.len != mask->enc_opts.len) { 1364 NL_SET_ERR_MSG(extack, "Key and mask miss aligned"); 1365 return -EINVAL; 1366 } 1367 1368 if (msk_depth) 1369 nla_opt_msk = nla_next(nla_opt_msk, &msk_depth); 1370 break; 1371 default: 1372 NL_SET_ERR_MSG(extack, "Unknown tunnel option type"); 1373 return -EINVAL; 1374 } 1375 } 1376 1377 return 0; 1378 } 1379 1380 static int fl_set_key_ct(struct nlattr **tb, 1381 struct flow_dissector_key_ct *key, 1382 struct flow_dissector_key_ct *mask, 1383 struct netlink_ext_ack *extack) 1384 { 1385 if (tb[TCA_FLOWER_KEY_CT_STATE]) { 1386 if (!IS_ENABLED(CONFIG_NF_CONNTRACK)) { 1387 NL_SET_ERR_MSG(extack, "Conntrack isn't enabled"); 1388 return -EOPNOTSUPP; 1389 } 1390 fl_set_key_val(tb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE, 1391 &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK, 1392 sizeof(key->ct_state)); 1393 } 1394 if (tb[TCA_FLOWER_KEY_CT_ZONE]) { 1395 if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) { 1396 NL_SET_ERR_MSG(extack, "Conntrack zones isn't enabled"); 1397 return -EOPNOTSUPP; 1398 } 1399 fl_set_key_val(tb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE, 1400 &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK, 1401 sizeof(key->ct_zone)); 1402 } 1403 if (tb[TCA_FLOWER_KEY_CT_MARK]) { 1404 if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) { 1405 NL_SET_ERR_MSG(extack, "Conntrack mark isn't enabled"); 1406 return -EOPNOTSUPP; 1407 } 1408 fl_set_key_val(tb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK, 1409 &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK, 1410 sizeof(key->ct_mark)); 1411 } 1412 if (tb[TCA_FLOWER_KEY_CT_LABELS]) { 1413 if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) { 1414 NL_SET_ERR_MSG(extack, "Conntrack labels aren't enabled"); 1415 return -EOPNOTSUPP; 1416 } 1417 fl_set_key_val(tb, key->ct_labels, TCA_FLOWER_KEY_CT_LABELS, 1418 mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK, 1419 sizeof(key->ct_labels)); 1420 } 1421 1422 return 0; 1423 } 1424 1425 static int fl_set_key(struct net *net, struct nlattr **tb, 1426 struct fl_flow_key *key, struct fl_flow_key *mask, 1427 struct netlink_ext_ack *extack) 1428 { 1429 __be16 ethertype; 1430 int ret = 0; 1431 1432 if (tb[TCA_FLOWER_INDEV]) { 1433 int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack); 1434 if (err < 0) 1435 return err; 1436 key->meta.ingress_ifindex = err; 1437 mask->meta.ingress_ifindex = 0xffffffff; 1438 } 1439 1440 fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 1441 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 1442 sizeof(key->eth.dst)); 1443 fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 1444 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 1445 sizeof(key->eth.src)); 1446 1447 if (tb[TCA_FLOWER_KEY_ETH_TYPE]) { 1448 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]); 1449 1450 if (eth_type_vlan(ethertype)) { 1451 fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID, 1452 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, 1453 &mask->vlan); 1454 1455 if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) { 1456 ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]); 1457 if (eth_type_vlan(ethertype)) { 1458 fl_set_key_vlan(tb, ethertype, 1459 TCA_FLOWER_KEY_CVLAN_ID, 1460 TCA_FLOWER_KEY_CVLAN_PRIO, 1461 &key->cvlan, &mask->cvlan); 1462 fl_set_key_val(tb, &key->basic.n_proto, 1463 TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 1464 &mask->basic.n_proto, 1465 TCA_FLOWER_UNSPEC, 1466 sizeof(key->basic.n_proto)); 1467 } else { 1468 key->basic.n_proto = ethertype; 1469 mask->basic.n_proto = cpu_to_be16(~0); 1470 } 1471 } 1472 } else { 1473 key->basic.n_proto = ethertype; 1474 mask->basic.n_proto = cpu_to_be16(~0); 1475 } 1476 } 1477 1478 if (key->basic.n_proto == htons(ETH_P_IP) || 1479 key->basic.n_proto == htons(ETH_P_IPV6)) { 1480 fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 1481 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 1482 sizeof(key->basic.ip_proto)); 1483 fl_set_key_ip(tb, false, &key->ip, &mask->ip); 1484 } 1485 1486 if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) { 1487 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1488 mask->control.addr_type = ~0; 1489 fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 1490 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 1491 sizeof(key->ipv4.src)); 1492 fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 1493 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 1494 sizeof(key->ipv4.dst)); 1495 } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) { 1496 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1497 mask->control.addr_type = ~0; 1498 fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 1499 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 1500 sizeof(key->ipv6.src)); 1501 fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 1502 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 1503 sizeof(key->ipv6.dst)); 1504 } 1505 1506 if (key->basic.ip_proto == IPPROTO_TCP) { 1507 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 1508 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 1509 sizeof(key->tp.src)); 1510 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 1511 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 1512 sizeof(key->tp.dst)); 1513 fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 1514 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 1515 sizeof(key->tcp.flags)); 1516 } else if (key->basic.ip_proto == IPPROTO_UDP) { 1517 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 1518 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 1519 sizeof(key->tp.src)); 1520 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 1521 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 1522 sizeof(key->tp.dst)); 1523 } else if (key->basic.ip_proto == IPPROTO_SCTP) { 1524 fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 1525 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 1526 sizeof(key->tp.src)); 1527 fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 1528 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 1529 sizeof(key->tp.dst)); 1530 } else if (key->basic.n_proto == htons(ETH_P_IP) && 1531 key->basic.ip_proto == IPPROTO_ICMP) { 1532 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE, 1533 &mask->icmp.type, 1534 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 1535 sizeof(key->icmp.type)); 1536 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE, 1537 &mask->icmp.code, 1538 TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 1539 sizeof(key->icmp.code)); 1540 } else if (key->basic.n_proto == htons(ETH_P_IPV6) && 1541 key->basic.ip_proto == IPPROTO_ICMPV6) { 1542 fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE, 1543 &mask->icmp.type, 1544 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 1545 sizeof(key->icmp.type)); 1546 fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE, 1547 &mask->icmp.code, 1548 TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 1549 sizeof(key->icmp.code)); 1550 } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) || 1551 key->basic.n_proto == htons(ETH_P_MPLS_MC)) { 1552 ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls, extack); 1553 if (ret) 1554 return ret; 1555 } else if (key->basic.n_proto == htons(ETH_P_ARP) || 1556 key->basic.n_proto == htons(ETH_P_RARP)) { 1557 fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP, 1558 &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK, 1559 sizeof(key->arp.sip)); 1560 fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP, 1561 &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK, 1562 sizeof(key->arp.tip)); 1563 fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP, 1564 &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK, 1565 sizeof(key->arp.op)); 1566 fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 1567 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 1568 sizeof(key->arp.sha)); 1569 fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 1570 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 1571 sizeof(key->arp.tha)); 1572 } 1573 1574 if (key->basic.ip_proto == IPPROTO_TCP || 1575 key->basic.ip_proto == IPPROTO_UDP || 1576 key->basic.ip_proto == IPPROTO_SCTP) { 1577 ret = fl_set_key_port_range(tb, key, mask, extack); 1578 if (ret) 1579 return ret; 1580 } 1581 1582 if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] || 1583 tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) { 1584 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 1585 mask->enc_control.addr_type = ~0; 1586 fl_set_key_val(tb, &key->enc_ipv4.src, 1587 TCA_FLOWER_KEY_ENC_IPV4_SRC, 1588 &mask->enc_ipv4.src, 1589 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 1590 sizeof(key->enc_ipv4.src)); 1591 fl_set_key_val(tb, &key->enc_ipv4.dst, 1592 TCA_FLOWER_KEY_ENC_IPV4_DST, 1593 &mask->enc_ipv4.dst, 1594 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 1595 sizeof(key->enc_ipv4.dst)); 1596 } 1597 1598 if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] || 1599 tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) { 1600 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 1601 mask->enc_control.addr_type = ~0; 1602 fl_set_key_val(tb, &key->enc_ipv6.src, 1603 TCA_FLOWER_KEY_ENC_IPV6_SRC, 1604 &mask->enc_ipv6.src, 1605 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 1606 sizeof(key->enc_ipv6.src)); 1607 fl_set_key_val(tb, &key->enc_ipv6.dst, 1608 TCA_FLOWER_KEY_ENC_IPV6_DST, 1609 &mask->enc_ipv6.dst, 1610 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 1611 sizeof(key->enc_ipv6.dst)); 1612 } 1613 1614 fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID, 1615 &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC, 1616 sizeof(key->enc_key_id.keyid)); 1617 1618 fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 1619 &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 1620 sizeof(key->enc_tp.src)); 1621 1622 fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 1623 &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 1624 sizeof(key->enc_tp.dst)); 1625 1626 fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip); 1627 1628 if (tb[TCA_FLOWER_KEY_ENC_OPTS]) { 1629 ret = fl_set_enc_opt(tb, key, mask, extack); 1630 if (ret) 1631 return ret; 1632 } 1633 1634 ret = fl_set_key_ct(tb, &key->ct, &mask->ct, extack); 1635 if (ret) 1636 return ret; 1637 1638 if (tb[TCA_FLOWER_KEY_FLAGS]) 1639 ret = fl_set_key_flags(tb, &key->control.flags, 1640 &mask->control.flags, extack); 1641 1642 return ret; 1643 } 1644 1645 static void fl_mask_copy(struct fl_flow_mask *dst, 1646 struct fl_flow_mask *src) 1647 { 1648 const void *psrc = fl_key_get_start(&src->key, src); 1649 void *pdst = fl_key_get_start(&dst->key, src); 1650 1651 memcpy(pdst, psrc, fl_mask_range(src)); 1652 dst->range = src->range; 1653 } 1654 1655 static const struct rhashtable_params fl_ht_params = { 1656 .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */ 1657 .head_offset = offsetof(struct cls_fl_filter, ht_node), 1658 .automatic_shrinking = true, 1659 }; 1660 1661 static int fl_init_mask_hashtable(struct fl_flow_mask *mask) 1662 { 1663 mask->filter_ht_params = fl_ht_params; 1664 mask->filter_ht_params.key_len = fl_mask_range(mask); 1665 mask->filter_ht_params.key_offset += mask->range.start; 1666 1667 return rhashtable_init(&mask->ht, &mask->filter_ht_params); 1668 } 1669 1670 #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member) 1671 #define FL_KEY_MEMBER_SIZE(member) sizeof_field(struct fl_flow_key, member) 1672 1673 #define FL_KEY_IS_MASKED(mask, member) \ 1674 memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \ 1675 0, FL_KEY_MEMBER_SIZE(member)) \ 1676 1677 #define FL_KEY_SET(keys, cnt, id, member) \ 1678 do { \ 1679 keys[cnt].key_id = id; \ 1680 keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \ 1681 cnt++; \ 1682 } while(0); 1683 1684 #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \ 1685 do { \ 1686 if (FL_KEY_IS_MASKED(mask, member)) \ 1687 FL_KEY_SET(keys, cnt, id, member); \ 1688 } while(0); 1689 1690 static void fl_init_dissector(struct flow_dissector *dissector, 1691 struct fl_flow_key *mask) 1692 { 1693 struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX]; 1694 size_t cnt = 0; 1695 1696 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1697 FLOW_DISSECTOR_KEY_META, meta); 1698 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control); 1699 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic); 1700 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1701 FLOW_DISSECTOR_KEY_ETH_ADDRS, eth); 1702 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1703 FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 1704 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1705 FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 1706 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1707 FLOW_DISSECTOR_KEY_PORTS, tp); 1708 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1709 FLOW_DISSECTOR_KEY_PORTS_RANGE, tp_range); 1710 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1711 FLOW_DISSECTOR_KEY_IP, ip); 1712 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1713 FLOW_DISSECTOR_KEY_TCP, tcp); 1714 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1715 FLOW_DISSECTOR_KEY_ICMP, icmp); 1716 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1717 FLOW_DISSECTOR_KEY_ARP, arp); 1718 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1719 FLOW_DISSECTOR_KEY_MPLS, mpls); 1720 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1721 FLOW_DISSECTOR_KEY_VLAN, vlan); 1722 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1723 FLOW_DISSECTOR_KEY_CVLAN, cvlan); 1724 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1725 FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 1726 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1727 FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4); 1728 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1729 FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6); 1730 if (FL_KEY_IS_MASKED(mask, enc_ipv4) || 1731 FL_KEY_IS_MASKED(mask, enc_ipv6)) 1732 FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL, 1733 enc_control); 1734 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1735 FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp); 1736 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1737 FLOW_DISSECTOR_KEY_ENC_IP, enc_ip); 1738 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1739 FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts); 1740 FL_KEY_SET_IF_MASKED(mask, keys, cnt, 1741 FLOW_DISSECTOR_KEY_CT, ct); 1742 1743 skb_flow_dissector_init(dissector, keys, cnt); 1744 } 1745 1746 static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head, 1747 struct fl_flow_mask *mask) 1748 { 1749 struct fl_flow_mask *newmask; 1750 int err; 1751 1752 newmask = kzalloc(sizeof(*newmask), GFP_KERNEL); 1753 if (!newmask) 1754 return ERR_PTR(-ENOMEM); 1755 1756 fl_mask_copy(newmask, mask); 1757 1758 if ((newmask->key.tp_range.tp_min.dst && 1759 newmask->key.tp_range.tp_max.dst) || 1760 (newmask->key.tp_range.tp_min.src && 1761 newmask->key.tp_range.tp_max.src)) 1762 newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE; 1763 1764 err = fl_init_mask_hashtable(newmask); 1765 if (err) 1766 goto errout_free; 1767 1768 fl_init_dissector(&newmask->dissector, &newmask->key); 1769 1770 INIT_LIST_HEAD_RCU(&newmask->filters); 1771 1772 refcount_set(&newmask->refcnt, 1); 1773 err = rhashtable_replace_fast(&head->ht, &mask->ht_node, 1774 &newmask->ht_node, mask_ht_params); 1775 if (err) 1776 goto errout_destroy; 1777 1778 spin_lock(&head->masks_lock); 1779 list_add_tail_rcu(&newmask->list, &head->masks); 1780 spin_unlock(&head->masks_lock); 1781 1782 return newmask; 1783 1784 errout_destroy: 1785 rhashtable_destroy(&newmask->ht); 1786 errout_free: 1787 kfree(newmask); 1788 1789 return ERR_PTR(err); 1790 } 1791 1792 static int fl_check_assign_mask(struct cls_fl_head *head, 1793 struct cls_fl_filter *fnew, 1794 struct cls_fl_filter *fold, 1795 struct fl_flow_mask *mask) 1796 { 1797 struct fl_flow_mask *newmask; 1798 int ret = 0; 1799 1800 rcu_read_lock(); 1801 1802 /* Insert mask as temporary node to prevent concurrent creation of mask 1803 * with same key. Any concurrent lookups with same key will return 1804 * -EAGAIN because mask's refcnt is zero. 1805 */ 1806 fnew->mask = rhashtable_lookup_get_insert_fast(&head->ht, 1807 &mask->ht_node, 1808 mask_ht_params); 1809 if (!fnew->mask) { 1810 rcu_read_unlock(); 1811 1812 if (fold) { 1813 ret = -EINVAL; 1814 goto errout_cleanup; 1815 } 1816 1817 newmask = fl_create_new_mask(head, mask); 1818 if (IS_ERR(newmask)) { 1819 ret = PTR_ERR(newmask); 1820 goto errout_cleanup; 1821 } 1822 1823 fnew->mask = newmask; 1824 return 0; 1825 } else if (IS_ERR(fnew->mask)) { 1826 ret = PTR_ERR(fnew->mask); 1827 } else if (fold && fold->mask != fnew->mask) { 1828 ret = -EINVAL; 1829 } else if (!refcount_inc_not_zero(&fnew->mask->refcnt)) { 1830 /* Mask was deleted concurrently, try again */ 1831 ret = -EAGAIN; 1832 } 1833 rcu_read_unlock(); 1834 return ret; 1835 1836 errout_cleanup: 1837 rhashtable_remove_fast(&head->ht, &mask->ht_node, 1838 mask_ht_params); 1839 return ret; 1840 } 1841 1842 static int fl_set_parms(struct net *net, struct tcf_proto *tp, 1843 struct cls_fl_filter *f, struct fl_flow_mask *mask, 1844 unsigned long base, struct nlattr **tb, 1845 struct nlattr *est, bool ovr, 1846 struct fl_flow_tmplt *tmplt, bool rtnl_held, 1847 struct netlink_ext_ack *extack) 1848 { 1849 int err; 1850 1851 err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, rtnl_held, 1852 extack); 1853 if (err < 0) 1854 return err; 1855 1856 if (tb[TCA_FLOWER_CLASSID]) { 1857 f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]); 1858 if (!rtnl_held) 1859 rtnl_lock(); 1860 tcf_bind_filter(tp, &f->res, base); 1861 if (!rtnl_held) 1862 rtnl_unlock(); 1863 } 1864 1865 err = fl_set_key(net, tb, &f->key, &mask->key, extack); 1866 if (err) 1867 return err; 1868 1869 fl_mask_update_range(mask); 1870 fl_set_masked_key(&f->mkey, &f->key, mask); 1871 1872 if (!fl_mask_fits_tmplt(tmplt, mask)) { 1873 NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template"); 1874 return -EINVAL; 1875 } 1876 1877 return 0; 1878 } 1879 1880 static int fl_ht_insert_unique(struct cls_fl_filter *fnew, 1881 struct cls_fl_filter *fold, 1882 bool *in_ht) 1883 { 1884 struct fl_flow_mask *mask = fnew->mask; 1885 int err; 1886 1887 err = rhashtable_lookup_insert_fast(&mask->ht, 1888 &fnew->ht_node, 1889 mask->filter_ht_params); 1890 if (err) { 1891 *in_ht = false; 1892 /* It is okay if filter with same key exists when 1893 * overwriting. 1894 */ 1895 return fold && err == -EEXIST ? 0 : err; 1896 } 1897 1898 *in_ht = true; 1899 return 0; 1900 } 1901 1902 static int fl_change(struct net *net, struct sk_buff *in_skb, 1903 struct tcf_proto *tp, unsigned long base, 1904 u32 handle, struct nlattr **tca, 1905 void **arg, bool ovr, bool rtnl_held, 1906 struct netlink_ext_ack *extack) 1907 { 1908 struct cls_fl_head *head = fl_head_dereference(tp); 1909 struct cls_fl_filter *fold = *arg; 1910 struct cls_fl_filter *fnew; 1911 struct fl_flow_mask *mask; 1912 struct nlattr **tb; 1913 bool in_ht; 1914 int err; 1915 1916 if (!tca[TCA_OPTIONS]) { 1917 err = -EINVAL; 1918 goto errout_fold; 1919 } 1920 1921 mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL); 1922 if (!mask) { 1923 err = -ENOBUFS; 1924 goto errout_fold; 1925 } 1926 1927 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 1928 if (!tb) { 1929 err = -ENOBUFS; 1930 goto errout_mask_alloc; 1931 } 1932 1933 err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 1934 tca[TCA_OPTIONS], fl_policy, NULL); 1935 if (err < 0) 1936 goto errout_tb; 1937 1938 if (fold && handle && fold->handle != handle) { 1939 err = -EINVAL; 1940 goto errout_tb; 1941 } 1942 1943 fnew = kzalloc(sizeof(*fnew), GFP_KERNEL); 1944 if (!fnew) { 1945 err = -ENOBUFS; 1946 goto errout_tb; 1947 } 1948 INIT_LIST_HEAD(&fnew->hw_list); 1949 refcount_set(&fnew->refcnt, 1); 1950 1951 err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0); 1952 if (err < 0) 1953 goto errout; 1954 1955 if (tb[TCA_FLOWER_FLAGS]) { 1956 fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]); 1957 1958 if (!tc_flags_valid(fnew->flags)) { 1959 err = -EINVAL; 1960 goto errout; 1961 } 1962 } 1963 1964 err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr, 1965 tp->chain->tmplt_priv, rtnl_held, extack); 1966 if (err) 1967 goto errout; 1968 1969 err = fl_check_assign_mask(head, fnew, fold, mask); 1970 if (err) 1971 goto errout; 1972 1973 err = fl_ht_insert_unique(fnew, fold, &in_ht); 1974 if (err) 1975 goto errout_mask; 1976 1977 if (!tc_skip_hw(fnew->flags)) { 1978 err = fl_hw_replace_filter(tp, fnew, rtnl_held, extack); 1979 if (err) 1980 goto errout_ht; 1981 } 1982 1983 if (!tc_in_hw(fnew->flags)) 1984 fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW; 1985 1986 spin_lock(&tp->lock); 1987 1988 /* tp was deleted concurrently. -EAGAIN will cause caller to lookup 1989 * proto again or create new one, if necessary. 1990 */ 1991 if (tp->deleting) { 1992 err = -EAGAIN; 1993 goto errout_hw; 1994 } 1995 1996 if (fold) { 1997 /* Fold filter was deleted concurrently. Retry lookup. */ 1998 if (fold->deleted) { 1999 err = -EAGAIN; 2000 goto errout_hw; 2001 } 2002 2003 fnew->handle = handle; 2004 2005 if (!in_ht) { 2006 struct rhashtable_params params = 2007 fnew->mask->filter_ht_params; 2008 2009 err = rhashtable_insert_fast(&fnew->mask->ht, 2010 &fnew->ht_node, 2011 params); 2012 if (err) 2013 goto errout_hw; 2014 in_ht = true; 2015 } 2016 2017 refcount_inc(&fnew->refcnt); 2018 rhashtable_remove_fast(&fold->mask->ht, 2019 &fold->ht_node, 2020 fold->mask->filter_ht_params); 2021 idr_replace(&head->handle_idr, fnew, fnew->handle); 2022 list_replace_rcu(&fold->list, &fnew->list); 2023 fold->deleted = true; 2024 2025 spin_unlock(&tp->lock); 2026 2027 fl_mask_put(head, fold->mask); 2028 if (!tc_skip_hw(fold->flags)) 2029 fl_hw_destroy_filter(tp, fold, rtnl_held, NULL); 2030 tcf_unbind_filter(tp, &fold->res); 2031 /* Caller holds reference to fold, so refcnt is always > 0 2032 * after this. 2033 */ 2034 refcount_dec(&fold->refcnt); 2035 __fl_put(fold); 2036 } else { 2037 if (handle) { 2038 /* user specifies a handle and it doesn't exist */ 2039 err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 2040 handle, GFP_ATOMIC); 2041 2042 /* Filter with specified handle was concurrently 2043 * inserted after initial check in cls_api. This is not 2044 * necessarily an error if NLM_F_EXCL is not set in 2045 * message flags. Returning EAGAIN will cause cls_api to 2046 * try to update concurrently inserted rule. 2047 */ 2048 if (err == -ENOSPC) 2049 err = -EAGAIN; 2050 } else { 2051 handle = 1; 2052 err = idr_alloc_u32(&head->handle_idr, fnew, &handle, 2053 INT_MAX, GFP_ATOMIC); 2054 } 2055 if (err) 2056 goto errout_hw; 2057 2058 refcount_inc(&fnew->refcnt); 2059 fnew->handle = handle; 2060 list_add_tail_rcu(&fnew->list, &fnew->mask->filters); 2061 spin_unlock(&tp->lock); 2062 } 2063 2064 *arg = fnew; 2065 2066 kfree(tb); 2067 tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 2068 return 0; 2069 2070 errout_ht: 2071 spin_lock(&tp->lock); 2072 errout_hw: 2073 fnew->deleted = true; 2074 spin_unlock(&tp->lock); 2075 if (!tc_skip_hw(fnew->flags)) 2076 fl_hw_destroy_filter(tp, fnew, rtnl_held, NULL); 2077 if (in_ht) 2078 rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node, 2079 fnew->mask->filter_ht_params); 2080 errout_mask: 2081 fl_mask_put(head, fnew->mask); 2082 errout: 2083 __fl_put(fnew); 2084 errout_tb: 2085 kfree(tb); 2086 errout_mask_alloc: 2087 tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work); 2088 errout_fold: 2089 if (fold) 2090 __fl_put(fold); 2091 return err; 2092 } 2093 2094 static int fl_delete(struct tcf_proto *tp, void *arg, bool *last, 2095 bool rtnl_held, struct netlink_ext_ack *extack) 2096 { 2097 struct cls_fl_head *head = fl_head_dereference(tp); 2098 struct cls_fl_filter *f = arg; 2099 bool last_on_mask; 2100 int err = 0; 2101 2102 err = __fl_delete(tp, f, &last_on_mask, rtnl_held, extack); 2103 *last = list_empty(&head->masks); 2104 __fl_put(f); 2105 2106 return err; 2107 } 2108 2109 static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg, 2110 bool rtnl_held) 2111 { 2112 struct cls_fl_head *head = fl_head_dereference(tp); 2113 unsigned long id = arg->cookie, tmp; 2114 struct cls_fl_filter *f; 2115 2116 arg->count = arg->skip; 2117 2118 idr_for_each_entry_continue_ul(&head->handle_idr, f, tmp, id) { 2119 /* don't return filters that are being deleted */ 2120 if (!refcount_inc_not_zero(&f->refcnt)) 2121 continue; 2122 if (arg->fn(tp, f, arg) < 0) { 2123 __fl_put(f); 2124 arg->stop = 1; 2125 break; 2126 } 2127 __fl_put(f); 2128 arg->count++; 2129 } 2130 arg->cookie = id; 2131 } 2132 2133 static struct cls_fl_filter * 2134 fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add) 2135 { 2136 struct cls_fl_head *head = fl_head_dereference(tp); 2137 2138 spin_lock(&tp->lock); 2139 if (list_empty(&head->hw_filters)) { 2140 spin_unlock(&tp->lock); 2141 return NULL; 2142 } 2143 2144 if (!f) 2145 f = list_entry(&head->hw_filters, struct cls_fl_filter, 2146 hw_list); 2147 list_for_each_entry_continue(f, &head->hw_filters, hw_list) { 2148 if (!(add && f->deleted) && refcount_inc_not_zero(&f->refcnt)) { 2149 spin_unlock(&tp->lock); 2150 return f; 2151 } 2152 } 2153 2154 spin_unlock(&tp->lock); 2155 return NULL; 2156 } 2157 2158 static int fl_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb, 2159 void *cb_priv, struct netlink_ext_ack *extack) 2160 { 2161 struct tcf_block *block = tp->chain->block; 2162 struct flow_cls_offload cls_flower = {}; 2163 struct cls_fl_filter *f = NULL; 2164 int err; 2165 2166 /* hw_filters list can only be changed by hw offload functions after 2167 * obtaining rtnl lock. Make sure it is not changed while reoffload is 2168 * iterating it. 2169 */ 2170 ASSERT_RTNL(); 2171 2172 while ((f = fl_get_next_hw_filter(tp, f, add))) { 2173 cls_flower.rule = 2174 flow_rule_alloc(tcf_exts_num_actions(&f->exts)); 2175 if (!cls_flower.rule) { 2176 __fl_put(f); 2177 return -ENOMEM; 2178 } 2179 2180 tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, 2181 extack); 2182 cls_flower.command = add ? 2183 FLOW_CLS_REPLACE : FLOW_CLS_DESTROY; 2184 cls_flower.cookie = (unsigned long)f; 2185 cls_flower.rule->match.dissector = &f->mask->dissector; 2186 cls_flower.rule->match.mask = &f->mask->key; 2187 cls_flower.rule->match.key = &f->mkey; 2188 2189 err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts); 2190 if (err) { 2191 kfree(cls_flower.rule); 2192 if (tc_skip_sw(f->flags)) { 2193 NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action"); 2194 __fl_put(f); 2195 return err; 2196 } 2197 goto next_flow; 2198 } 2199 2200 cls_flower.classid = f->res.classid; 2201 2202 err = tc_setup_cb_reoffload(block, tp, add, cb, 2203 TC_SETUP_CLSFLOWER, &cls_flower, 2204 cb_priv, &f->flags, 2205 &f->in_hw_count); 2206 tc_cleanup_flow_action(&cls_flower.rule->action); 2207 kfree(cls_flower.rule); 2208 2209 if (err) { 2210 __fl_put(f); 2211 return err; 2212 } 2213 next_flow: 2214 __fl_put(f); 2215 } 2216 2217 return 0; 2218 } 2219 2220 static void fl_hw_add(struct tcf_proto *tp, void *type_data) 2221 { 2222 struct flow_cls_offload *cls_flower = type_data; 2223 struct cls_fl_filter *f = 2224 (struct cls_fl_filter *) cls_flower->cookie; 2225 struct cls_fl_head *head = fl_head_dereference(tp); 2226 2227 spin_lock(&tp->lock); 2228 list_add(&f->hw_list, &head->hw_filters); 2229 spin_unlock(&tp->lock); 2230 } 2231 2232 static void fl_hw_del(struct tcf_proto *tp, void *type_data) 2233 { 2234 struct flow_cls_offload *cls_flower = type_data; 2235 struct cls_fl_filter *f = 2236 (struct cls_fl_filter *) cls_flower->cookie; 2237 2238 spin_lock(&tp->lock); 2239 if (!list_empty(&f->hw_list)) 2240 list_del_init(&f->hw_list); 2241 spin_unlock(&tp->lock); 2242 } 2243 2244 static int fl_hw_create_tmplt(struct tcf_chain *chain, 2245 struct fl_flow_tmplt *tmplt) 2246 { 2247 struct flow_cls_offload cls_flower = {}; 2248 struct tcf_block *block = chain->block; 2249 2250 cls_flower.rule = flow_rule_alloc(0); 2251 if (!cls_flower.rule) 2252 return -ENOMEM; 2253 2254 cls_flower.common.chain_index = chain->index; 2255 cls_flower.command = FLOW_CLS_TMPLT_CREATE; 2256 cls_flower.cookie = (unsigned long) tmplt; 2257 cls_flower.rule->match.dissector = &tmplt->dissector; 2258 cls_flower.rule->match.mask = &tmplt->mask; 2259 cls_flower.rule->match.key = &tmplt->dummy_key; 2260 2261 /* We don't care if driver (any of them) fails to handle this 2262 * call. It serves just as a hint for it. 2263 */ 2264 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true); 2265 kfree(cls_flower.rule); 2266 2267 return 0; 2268 } 2269 2270 static void fl_hw_destroy_tmplt(struct tcf_chain *chain, 2271 struct fl_flow_tmplt *tmplt) 2272 { 2273 struct flow_cls_offload cls_flower = {}; 2274 struct tcf_block *block = chain->block; 2275 2276 cls_flower.common.chain_index = chain->index; 2277 cls_flower.command = FLOW_CLS_TMPLT_DESTROY; 2278 cls_flower.cookie = (unsigned long) tmplt; 2279 2280 tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true); 2281 } 2282 2283 static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain, 2284 struct nlattr **tca, 2285 struct netlink_ext_ack *extack) 2286 { 2287 struct fl_flow_tmplt *tmplt; 2288 struct nlattr **tb; 2289 int err; 2290 2291 if (!tca[TCA_OPTIONS]) 2292 return ERR_PTR(-EINVAL); 2293 2294 tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL); 2295 if (!tb) 2296 return ERR_PTR(-ENOBUFS); 2297 err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX, 2298 tca[TCA_OPTIONS], fl_policy, NULL); 2299 if (err) 2300 goto errout_tb; 2301 2302 tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL); 2303 if (!tmplt) { 2304 err = -ENOMEM; 2305 goto errout_tb; 2306 } 2307 tmplt->chain = chain; 2308 err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack); 2309 if (err) 2310 goto errout_tmplt; 2311 2312 fl_init_dissector(&tmplt->dissector, &tmplt->mask); 2313 2314 err = fl_hw_create_tmplt(chain, tmplt); 2315 if (err) 2316 goto errout_tmplt; 2317 2318 kfree(tb); 2319 return tmplt; 2320 2321 errout_tmplt: 2322 kfree(tmplt); 2323 errout_tb: 2324 kfree(tb); 2325 return ERR_PTR(err); 2326 } 2327 2328 static void fl_tmplt_destroy(void *tmplt_priv) 2329 { 2330 struct fl_flow_tmplt *tmplt = tmplt_priv; 2331 2332 fl_hw_destroy_tmplt(tmplt->chain, tmplt); 2333 kfree(tmplt); 2334 } 2335 2336 static int fl_dump_key_val(struct sk_buff *skb, 2337 void *val, int val_type, 2338 void *mask, int mask_type, int len) 2339 { 2340 int err; 2341 2342 if (!memchr_inv(mask, 0, len)) 2343 return 0; 2344 err = nla_put(skb, val_type, len, val); 2345 if (err) 2346 return err; 2347 if (mask_type != TCA_FLOWER_UNSPEC) { 2348 err = nla_put(skb, mask_type, len, mask); 2349 if (err) 2350 return err; 2351 } 2352 return 0; 2353 } 2354 2355 static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key, 2356 struct fl_flow_key *mask) 2357 { 2358 if (fl_dump_key_val(skb, &key->tp_range.tp_min.dst, 2359 TCA_FLOWER_KEY_PORT_DST_MIN, 2360 &mask->tp_range.tp_min.dst, TCA_FLOWER_UNSPEC, 2361 sizeof(key->tp_range.tp_min.dst)) || 2362 fl_dump_key_val(skb, &key->tp_range.tp_max.dst, 2363 TCA_FLOWER_KEY_PORT_DST_MAX, 2364 &mask->tp_range.tp_max.dst, TCA_FLOWER_UNSPEC, 2365 sizeof(key->tp_range.tp_max.dst)) || 2366 fl_dump_key_val(skb, &key->tp_range.tp_min.src, 2367 TCA_FLOWER_KEY_PORT_SRC_MIN, 2368 &mask->tp_range.tp_min.src, TCA_FLOWER_UNSPEC, 2369 sizeof(key->tp_range.tp_min.src)) || 2370 fl_dump_key_val(skb, &key->tp_range.tp_max.src, 2371 TCA_FLOWER_KEY_PORT_SRC_MAX, 2372 &mask->tp_range.tp_max.src, TCA_FLOWER_UNSPEC, 2373 sizeof(key->tp_range.tp_max.src))) 2374 return -1; 2375 2376 return 0; 2377 } 2378 2379 static int fl_dump_key_mpls_opt_lse(struct sk_buff *skb, 2380 struct flow_dissector_key_mpls *mpls_key, 2381 struct flow_dissector_key_mpls *mpls_mask, 2382 u8 lse_index) 2383 { 2384 struct flow_dissector_mpls_lse *lse_mask = &mpls_mask->ls[lse_index]; 2385 struct flow_dissector_mpls_lse *lse_key = &mpls_key->ls[lse_index]; 2386 int err; 2387 2388 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH, 2389 lse_index + 1); 2390 if (err) 2391 return err; 2392 2393 if (lse_mask->mpls_ttl) { 2394 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL, 2395 lse_key->mpls_ttl); 2396 if (err) 2397 return err; 2398 } 2399 if (lse_mask->mpls_bos) { 2400 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS, 2401 lse_key->mpls_bos); 2402 if (err) 2403 return err; 2404 } 2405 if (lse_mask->mpls_tc) { 2406 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_TC, 2407 lse_key->mpls_tc); 2408 if (err) 2409 return err; 2410 } 2411 if (lse_mask->mpls_label) { 2412 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL, 2413 lse_key->mpls_label); 2414 if (err) 2415 return err; 2416 } 2417 2418 return 0; 2419 } 2420 2421 static int fl_dump_key_mpls_opts(struct sk_buff *skb, 2422 struct flow_dissector_key_mpls *mpls_key, 2423 struct flow_dissector_key_mpls *mpls_mask) 2424 { 2425 struct nlattr *opts; 2426 struct nlattr *lse; 2427 u8 lse_index; 2428 int err; 2429 2430 opts = nla_nest_start(skb, TCA_FLOWER_KEY_MPLS_OPTS); 2431 if (!opts) 2432 return -EMSGSIZE; 2433 2434 for (lse_index = 0; lse_index < FLOW_DIS_MPLS_MAX; lse_index++) { 2435 if (!(mpls_mask->used_lses & 1 << lse_index)) 2436 continue; 2437 2438 lse = nla_nest_start(skb, TCA_FLOWER_KEY_MPLS_OPTS_LSE); 2439 if (!lse) { 2440 err = -EMSGSIZE; 2441 goto err_opts; 2442 } 2443 2444 err = fl_dump_key_mpls_opt_lse(skb, mpls_key, mpls_mask, 2445 lse_index); 2446 if (err) 2447 goto err_opts_lse; 2448 nla_nest_end(skb, lse); 2449 } 2450 nla_nest_end(skb, opts); 2451 2452 return 0; 2453 2454 err_opts_lse: 2455 nla_nest_cancel(skb, lse); 2456 err_opts: 2457 nla_nest_cancel(skb, opts); 2458 2459 return err; 2460 } 2461 2462 static int fl_dump_key_mpls(struct sk_buff *skb, 2463 struct flow_dissector_key_mpls *mpls_key, 2464 struct flow_dissector_key_mpls *mpls_mask) 2465 { 2466 struct flow_dissector_mpls_lse *lse_mask; 2467 struct flow_dissector_mpls_lse *lse_key; 2468 int err; 2469 2470 if (!mpls_mask->used_lses) 2471 return 0; 2472 2473 lse_mask = &mpls_mask->ls[0]; 2474 lse_key = &mpls_key->ls[0]; 2475 2476 /* For backward compatibility, don't use the MPLS nested attributes if 2477 * the rule can be expressed using the old attributes. 2478 */ 2479 if (mpls_mask->used_lses & ~1 || 2480 (!lse_mask->mpls_ttl && !lse_mask->mpls_bos && 2481 !lse_mask->mpls_tc && !lse_mask->mpls_label)) 2482 return fl_dump_key_mpls_opts(skb, mpls_key, mpls_mask); 2483 2484 if (lse_mask->mpls_ttl) { 2485 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL, 2486 lse_key->mpls_ttl); 2487 if (err) 2488 return err; 2489 } 2490 if (lse_mask->mpls_tc) { 2491 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC, 2492 lse_key->mpls_tc); 2493 if (err) 2494 return err; 2495 } 2496 if (lse_mask->mpls_label) { 2497 err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL, 2498 lse_key->mpls_label); 2499 if (err) 2500 return err; 2501 } 2502 if (lse_mask->mpls_bos) { 2503 err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS, 2504 lse_key->mpls_bos); 2505 if (err) 2506 return err; 2507 } 2508 return 0; 2509 } 2510 2511 static int fl_dump_key_ip(struct sk_buff *skb, bool encap, 2512 struct flow_dissector_key_ip *key, 2513 struct flow_dissector_key_ip *mask) 2514 { 2515 int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS; 2516 int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL; 2517 int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK; 2518 int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK; 2519 2520 if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) || 2521 fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl))) 2522 return -1; 2523 2524 return 0; 2525 } 2526 2527 static int fl_dump_key_vlan(struct sk_buff *skb, 2528 int vlan_id_key, int vlan_prio_key, 2529 struct flow_dissector_key_vlan *vlan_key, 2530 struct flow_dissector_key_vlan *vlan_mask) 2531 { 2532 int err; 2533 2534 if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask))) 2535 return 0; 2536 if (vlan_mask->vlan_id) { 2537 err = nla_put_u16(skb, vlan_id_key, 2538 vlan_key->vlan_id); 2539 if (err) 2540 return err; 2541 } 2542 if (vlan_mask->vlan_priority) { 2543 err = nla_put_u8(skb, vlan_prio_key, 2544 vlan_key->vlan_priority); 2545 if (err) 2546 return err; 2547 } 2548 return 0; 2549 } 2550 2551 static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask, 2552 u32 *flower_key, u32 *flower_mask, 2553 u32 flower_flag_bit, u32 dissector_flag_bit) 2554 { 2555 if (dissector_mask & dissector_flag_bit) { 2556 *flower_mask |= flower_flag_bit; 2557 if (dissector_key & dissector_flag_bit) 2558 *flower_key |= flower_flag_bit; 2559 } 2560 } 2561 2562 static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask) 2563 { 2564 u32 key, mask; 2565 __be32 _key, _mask; 2566 int err; 2567 2568 if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask))) 2569 return 0; 2570 2571 key = 0; 2572 mask = 0; 2573 2574 fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2575 TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT); 2576 fl_get_key_flag(flags_key, flags_mask, &key, &mask, 2577 TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST, 2578 FLOW_DIS_FIRST_FRAG); 2579 2580 _key = cpu_to_be32(key); 2581 _mask = cpu_to_be32(mask); 2582 2583 err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key); 2584 if (err) 2585 return err; 2586 2587 return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask); 2588 } 2589 2590 static int fl_dump_key_geneve_opt(struct sk_buff *skb, 2591 struct flow_dissector_key_enc_opts *enc_opts) 2592 { 2593 struct geneve_opt *opt; 2594 struct nlattr *nest; 2595 int opt_off = 0; 2596 2597 nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE); 2598 if (!nest) 2599 goto nla_put_failure; 2600 2601 while (enc_opts->len > opt_off) { 2602 opt = (struct geneve_opt *)&enc_opts->data[opt_off]; 2603 2604 if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS, 2605 opt->opt_class)) 2606 goto nla_put_failure; 2607 if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE, 2608 opt->type)) 2609 goto nla_put_failure; 2610 if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA, 2611 opt->length * 4, opt->opt_data)) 2612 goto nla_put_failure; 2613 2614 opt_off += sizeof(struct geneve_opt) + opt->length * 4; 2615 } 2616 nla_nest_end(skb, nest); 2617 return 0; 2618 2619 nla_put_failure: 2620 nla_nest_cancel(skb, nest); 2621 return -EMSGSIZE; 2622 } 2623 2624 static int fl_dump_key_vxlan_opt(struct sk_buff *skb, 2625 struct flow_dissector_key_enc_opts *enc_opts) 2626 { 2627 struct vxlan_metadata *md; 2628 struct nlattr *nest; 2629 2630 nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_VXLAN); 2631 if (!nest) 2632 goto nla_put_failure; 2633 2634 md = (struct vxlan_metadata *)&enc_opts->data[0]; 2635 if (nla_put_u32(skb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP, md->gbp)) 2636 goto nla_put_failure; 2637 2638 nla_nest_end(skb, nest); 2639 return 0; 2640 2641 nla_put_failure: 2642 nla_nest_cancel(skb, nest); 2643 return -EMSGSIZE; 2644 } 2645 2646 static int fl_dump_key_erspan_opt(struct sk_buff *skb, 2647 struct flow_dissector_key_enc_opts *enc_opts) 2648 { 2649 struct erspan_metadata *md; 2650 struct nlattr *nest; 2651 2652 nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_ERSPAN); 2653 if (!nest) 2654 goto nla_put_failure; 2655 2656 md = (struct erspan_metadata *)&enc_opts->data[0]; 2657 if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER, md->version)) 2658 goto nla_put_failure; 2659 2660 if (md->version == 1 && 2661 nla_put_be32(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX, md->u.index)) 2662 goto nla_put_failure; 2663 2664 if (md->version == 2 && 2665 (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR, 2666 md->u.md2.dir) || 2667 nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID, 2668 get_hwid(&md->u.md2)))) 2669 goto nla_put_failure; 2670 2671 nla_nest_end(skb, nest); 2672 return 0; 2673 2674 nla_put_failure: 2675 nla_nest_cancel(skb, nest); 2676 return -EMSGSIZE; 2677 } 2678 2679 static int fl_dump_key_ct(struct sk_buff *skb, 2680 struct flow_dissector_key_ct *key, 2681 struct flow_dissector_key_ct *mask) 2682 { 2683 if (IS_ENABLED(CONFIG_NF_CONNTRACK) && 2684 fl_dump_key_val(skb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE, 2685 &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK, 2686 sizeof(key->ct_state))) 2687 goto nla_put_failure; 2688 2689 if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) && 2690 fl_dump_key_val(skb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE, 2691 &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK, 2692 sizeof(key->ct_zone))) 2693 goto nla_put_failure; 2694 2695 if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) && 2696 fl_dump_key_val(skb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK, 2697 &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK, 2698 sizeof(key->ct_mark))) 2699 goto nla_put_failure; 2700 2701 if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) && 2702 fl_dump_key_val(skb, &key->ct_labels, TCA_FLOWER_KEY_CT_LABELS, 2703 &mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK, 2704 sizeof(key->ct_labels))) 2705 goto nla_put_failure; 2706 2707 return 0; 2708 2709 nla_put_failure: 2710 return -EMSGSIZE; 2711 } 2712 2713 static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type, 2714 struct flow_dissector_key_enc_opts *enc_opts) 2715 { 2716 struct nlattr *nest; 2717 int err; 2718 2719 if (!enc_opts->len) 2720 return 0; 2721 2722 nest = nla_nest_start_noflag(skb, enc_opt_type); 2723 if (!nest) 2724 goto nla_put_failure; 2725 2726 switch (enc_opts->dst_opt_type) { 2727 case TUNNEL_GENEVE_OPT: 2728 err = fl_dump_key_geneve_opt(skb, enc_opts); 2729 if (err) 2730 goto nla_put_failure; 2731 break; 2732 case TUNNEL_VXLAN_OPT: 2733 err = fl_dump_key_vxlan_opt(skb, enc_opts); 2734 if (err) 2735 goto nla_put_failure; 2736 break; 2737 case TUNNEL_ERSPAN_OPT: 2738 err = fl_dump_key_erspan_opt(skb, enc_opts); 2739 if (err) 2740 goto nla_put_failure; 2741 break; 2742 default: 2743 goto nla_put_failure; 2744 } 2745 nla_nest_end(skb, nest); 2746 return 0; 2747 2748 nla_put_failure: 2749 nla_nest_cancel(skb, nest); 2750 return -EMSGSIZE; 2751 } 2752 2753 static int fl_dump_key_enc_opt(struct sk_buff *skb, 2754 struct flow_dissector_key_enc_opts *key_opts, 2755 struct flow_dissector_key_enc_opts *msk_opts) 2756 { 2757 int err; 2758 2759 err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts); 2760 if (err) 2761 return err; 2762 2763 return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts); 2764 } 2765 2766 static int fl_dump_key(struct sk_buff *skb, struct net *net, 2767 struct fl_flow_key *key, struct fl_flow_key *mask) 2768 { 2769 if (mask->meta.ingress_ifindex) { 2770 struct net_device *dev; 2771 2772 dev = __dev_get_by_index(net, key->meta.ingress_ifindex); 2773 if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name)) 2774 goto nla_put_failure; 2775 } 2776 2777 if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST, 2778 mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK, 2779 sizeof(key->eth.dst)) || 2780 fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC, 2781 mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK, 2782 sizeof(key->eth.src)) || 2783 fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE, 2784 &mask->basic.n_proto, TCA_FLOWER_UNSPEC, 2785 sizeof(key->basic.n_proto))) 2786 goto nla_put_failure; 2787 2788 if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls)) 2789 goto nla_put_failure; 2790 2791 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID, 2792 TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan)) 2793 goto nla_put_failure; 2794 2795 if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID, 2796 TCA_FLOWER_KEY_CVLAN_PRIO, 2797 &key->cvlan, &mask->cvlan) || 2798 (mask->cvlan.vlan_tpid && 2799 nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2800 key->cvlan.vlan_tpid))) 2801 goto nla_put_failure; 2802 2803 if (mask->basic.n_proto) { 2804 if (mask->cvlan.vlan_tpid) { 2805 if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE, 2806 key->basic.n_proto)) 2807 goto nla_put_failure; 2808 } else if (mask->vlan.vlan_tpid) { 2809 if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE, 2810 key->basic.n_proto)) 2811 goto nla_put_failure; 2812 } 2813 } 2814 2815 if ((key->basic.n_proto == htons(ETH_P_IP) || 2816 key->basic.n_proto == htons(ETH_P_IPV6)) && 2817 (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO, 2818 &mask->basic.ip_proto, TCA_FLOWER_UNSPEC, 2819 sizeof(key->basic.ip_proto)) || 2820 fl_dump_key_ip(skb, false, &key->ip, &mask->ip))) 2821 goto nla_put_failure; 2822 2823 if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2824 (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC, 2825 &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK, 2826 sizeof(key->ipv4.src)) || 2827 fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST, 2828 &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK, 2829 sizeof(key->ipv4.dst)))) 2830 goto nla_put_failure; 2831 else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2832 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC, 2833 &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK, 2834 sizeof(key->ipv6.src)) || 2835 fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST, 2836 &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK, 2837 sizeof(key->ipv6.dst)))) 2838 goto nla_put_failure; 2839 2840 if (key->basic.ip_proto == IPPROTO_TCP && 2841 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC, 2842 &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK, 2843 sizeof(key->tp.src)) || 2844 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST, 2845 &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK, 2846 sizeof(key->tp.dst)) || 2847 fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS, 2848 &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK, 2849 sizeof(key->tcp.flags)))) 2850 goto nla_put_failure; 2851 else if (key->basic.ip_proto == IPPROTO_UDP && 2852 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC, 2853 &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK, 2854 sizeof(key->tp.src)) || 2855 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST, 2856 &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK, 2857 sizeof(key->tp.dst)))) 2858 goto nla_put_failure; 2859 else if (key->basic.ip_proto == IPPROTO_SCTP && 2860 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC, 2861 &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK, 2862 sizeof(key->tp.src)) || 2863 fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST, 2864 &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK, 2865 sizeof(key->tp.dst)))) 2866 goto nla_put_failure; 2867 else if (key->basic.n_proto == htons(ETH_P_IP) && 2868 key->basic.ip_proto == IPPROTO_ICMP && 2869 (fl_dump_key_val(skb, &key->icmp.type, 2870 TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type, 2871 TCA_FLOWER_KEY_ICMPV4_TYPE_MASK, 2872 sizeof(key->icmp.type)) || 2873 fl_dump_key_val(skb, &key->icmp.code, 2874 TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code, 2875 TCA_FLOWER_KEY_ICMPV4_CODE_MASK, 2876 sizeof(key->icmp.code)))) 2877 goto nla_put_failure; 2878 else if (key->basic.n_proto == htons(ETH_P_IPV6) && 2879 key->basic.ip_proto == IPPROTO_ICMPV6 && 2880 (fl_dump_key_val(skb, &key->icmp.type, 2881 TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type, 2882 TCA_FLOWER_KEY_ICMPV6_TYPE_MASK, 2883 sizeof(key->icmp.type)) || 2884 fl_dump_key_val(skb, &key->icmp.code, 2885 TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code, 2886 TCA_FLOWER_KEY_ICMPV6_CODE_MASK, 2887 sizeof(key->icmp.code)))) 2888 goto nla_put_failure; 2889 else if ((key->basic.n_proto == htons(ETH_P_ARP) || 2890 key->basic.n_proto == htons(ETH_P_RARP)) && 2891 (fl_dump_key_val(skb, &key->arp.sip, 2892 TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip, 2893 TCA_FLOWER_KEY_ARP_SIP_MASK, 2894 sizeof(key->arp.sip)) || 2895 fl_dump_key_val(skb, &key->arp.tip, 2896 TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip, 2897 TCA_FLOWER_KEY_ARP_TIP_MASK, 2898 sizeof(key->arp.tip)) || 2899 fl_dump_key_val(skb, &key->arp.op, 2900 TCA_FLOWER_KEY_ARP_OP, &mask->arp.op, 2901 TCA_FLOWER_KEY_ARP_OP_MASK, 2902 sizeof(key->arp.op)) || 2903 fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA, 2904 mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK, 2905 sizeof(key->arp.sha)) || 2906 fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA, 2907 mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK, 2908 sizeof(key->arp.tha)))) 2909 goto nla_put_failure; 2910 2911 if ((key->basic.ip_proto == IPPROTO_TCP || 2912 key->basic.ip_proto == IPPROTO_UDP || 2913 key->basic.ip_proto == IPPROTO_SCTP) && 2914 fl_dump_key_port_range(skb, key, mask)) 2915 goto nla_put_failure; 2916 2917 if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS && 2918 (fl_dump_key_val(skb, &key->enc_ipv4.src, 2919 TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src, 2920 TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK, 2921 sizeof(key->enc_ipv4.src)) || 2922 fl_dump_key_val(skb, &key->enc_ipv4.dst, 2923 TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst, 2924 TCA_FLOWER_KEY_ENC_IPV4_DST_MASK, 2925 sizeof(key->enc_ipv4.dst)))) 2926 goto nla_put_failure; 2927 else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS && 2928 (fl_dump_key_val(skb, &key->enc_ipv6.src, 2929 TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src, 2930 TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK, 2931 sizeof(key->enc_ipv6.src)) || 2932 fl_dump_key_val(skb, &key->enc_ipv6.dst, 2933 TCA_FLOWER_KEY_ENC_IPV6_DST, 2934 &mask->enc_ipv6.dst, 2935 TCA_FLOWER_KEY_ENC_IPV6_DST_MASK, 2936 sizeof(key->enc_ipv6.dst)))) 2937 goto nla_put_failure; 2938 2939 if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID, 2940 &mask->enc_key_id, TCA_FLOWER_UNSPEC, 2941 sizeof(key->enc_key_id)) || 2942 fl_dump_key_val(skb, &key->enc_tp.src, 2943 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT, 2944 &mask->enc_tp.src, 2945 TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK, 2946 sizeof(key->enc_tp.src)) || 2947 fl_dump_key_val(skb, &key->enc_tp.dst, 2948 TCA_FLOWER_KEY_ENC_UDP_DST_PORT, 2949 &mask->enc_tp.dst, 2950 TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK, 2951 sizeof(key->enc_tp.dst)) || 2952 fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) || 2953 fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts)) 2954 goto nla_put_failure; 2955 2956 if (fl_dump_key_ct(skb, &key->ct, &mask->ct)) 2957 goto nla_put_failure; 2958 2959 if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags)) 2960 goto nla_put_failure; 2961 2962 return 0; 2963 2964 nla_put_failure: 2965 return -EMSGSIZE; 2966 } 2967 2968 static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh, 2969 struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 2970 { 2971 struct cls_fl_filter *f = fh; 2972 struct nlattr *nest; 2973 struct fl_flow_key *key, *mask; 2974 bool skip_hw; 2975 2976 if (!f) 2977 return skb->len; 2978 2979 t->tcm_handle = f->handle; 2980 2981 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 2982 if (!nest) 2983 goto nla_put_failure; 2984 2985 spin_lock(&tp->lock); 2986 2987 if (f->res.classid && 2988 nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid)) 2989 goto nla_put_failure_locked; 2990 2991 key = &f->key; 2992 mask = &f->mask->key; 2993 skip_hw = tc_skip_hw(f->flags); 2994 2995 if (fl_dump_key(skb, net, key, mask)) 2996 goto nla_put_failure_locked; 2997 2998 if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags)) 2999 goto nla_put_failure_locked; 3000 3001 spin_unlock(&tp->lock); 3002 3003 if (!skip_hw) 3004 fl_hw_update_stats(tp, f, rtnl_held); 3005 3006 if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count)) 3007 goto nla_put_failure; 3008 3009 if (tcf_exts_dump(skb, &f->exts)) 3010 goto nla_put_failure; 3011 3012 nla_nest_end(skb, nest); 3013 3014 if (tcf_exts_dump_stats(skb, &f->exts) < 0) 3015 goto nla_put_failure; 3016 3017 return skb->len; 3018 3019 nla_put_failure_locked: 3020 spin_unlock(&tp->lock); 3021 nla_put_failure: 3022 nla_nest_cancel(skb, nest); 3023 return -1; 3024 } 3025 3026 static int fl_terse_dump(struct net *net, struct tcf_proto *tp, void *fh, 3027 struct sk_buff *skb, struct tcmsg *t, bool rtnl_held) 3028 { 3029 struct cls_fl_filter *f = fh; 3030 struct nlattr *nest; 3031 bool skip_hw; 3032 3033 if (!f) 3034 return skb->len; 3035 3036 t->tcm_handle = f->handle; 3037 3038 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 3039 if (!nest) 3040 goto nla_put_failure; 3041 3042 spin_lock(&tp->lock); 3043 3044 skip_hw = tc_skip_hw(f->flags); 3045 3046 if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags)) 3047 goto nla_put_failure_locked; 3048 3049 spin_unlock(&tp->lock); 3050 3051 if (!skip_hw) 3052 fl_hw_update_stats(tp, f, rtnl_held); 3053 3054 if (tcf_exts_terse_dump(skb, &f->exts)) 3055 goto nla_put_failure; 3056 3057 nla_nest_end(skb, nest); 3058 3059 return skb->len; 3060 3061 nla_put_failure_locked: 3062 spin_unlock(&tp->lock); 3063 nla_put_failure: 3064 nla_nest_cancel(skb, nest); 3065 return -1; 3066 } 3067 3068 static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv) 3069 { 3070 struct fl_flow_tmplt *tmplt = tmplt_priv; 3071 struct fl_flow_key *key, *mask; 3072 struct nlattr *nest; 3073 3074 nest = nla_nest_start_noflag(skb, TCA_OPTIONS); 3075 if (!nest) 3076 goto nla_put_failure; 3077 3078 key = &tmplt->dummy_key; 3079 mask = &tmplt->mask; 3080 3081 if (fl_dump_key(skb, net, key, mask)) 3082 goto nla_put_failure; 3083 3084 nla_nest_end(skb, nest); 3085 3086 return skb->len; 3087 3088 nla_put_failure: 3089 nla_nest_cancel(skb, nest); 3090 return -EMSGSIZE; 3091 } 3092 3093 static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q, 3094 unsigned long base) 3095 { 3096 struct cls_fl_filter *f = fh; 3097 3098 if (f && f->res.classid == classid) { 3099 if (cl) 3100 __tcf_bind_filter(q, &f->res, base); 3101 else 3102 __tcf_unbind_filter(q, &f->res); 3103 } 3104 } 3105 3106 static bool fl_delete_empty(struct tcf_proto *tp) 3107 { 3108 struct cls_fl_head *head = fl_head_dereference(tp); 3109 3110 spin_lock(&tp->lock); 3111 tp->deleting = idr_is_empty(&head->handle_idr); 3112 spin_unlock(&tp->lock); 3113 3114 return tp->deleting; 3115 } 3116 3117 static struct tcf_proto_ops cls_fl_ops __read_mostly = { 3118 .kind = "flower", 3119 .classify = fl_classify, 3120 .init = fl_init, 3121 .destroy = fl_destroy, 3122 .get = fl_get, 3123 .put = fl_put, 3124 .change = fl_change, 3125 .delete = fl_delete, 3126 .delete_empty = fl_delete_empty, 3127 .walk = fl_walk, 3128 .reoffload = fl_reoffload, 3129 .hw_add = fl_hw_add, 3130 .hw_del = fl_hw_del, 3131 .dump = fl_dump, 3132 .terse_dump = fl_terse_dump, 3133 .bind_class = fl_bind_class, 3134 .tmplt_create = fl_tmplt_create, 3135 .tmplt_destroy = fl_tmplt_destroy, 3136 .tmplt_dump = fl_tmplt_dump, 3137 .owner = THIS_MODULE, 3138 .flags = TCF_PROTO_OPS_DOIT_UNLOCKED, 3139 }; 3140 3141 static int __init cls_fl_init(void) 3142 { 3143 return register_tcf_proto_ops(&cls_fl_ops); 3144 } 3145 3146 static void __exit cls_fl_exit(void) 3147 { 3148 unregister_tcf_proto_ops(&cls_fl_ops); 3149 } 3150 3151 module_init(cls_fl_init); 3152 module_exit(cls_fl_exit); 3153 3154 MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); 3155 MODULE_DESCRIPTION("Flower classifier"); 3156 MODULE_LICENSE("GPL v2"); 3157