1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 9 struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13 }; 14 15 struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17 }; 18 19 struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21 }; 22 23 struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25 }; 26 27 struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29 }; 30 31 struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33 }; 34 35 struct flow_match_ipv4_addrs { 36 struct flow_dissector_key_ipv4_addrs *key, *mask; 37 }; 38 39 struct flow_match_ipv6_addrs { 40 struct flow_dissector_key_ipv6_addrs *key, *mask; 41 }; 42 43 struct flow_match_ip { 44 struct flow_dissector_key_ip *key, *mask; 45 }; 46 47 struct flow_match_ports { 48 struct flow_dissector_key_ports *key, *mask; 49 }; 50 51 struct flow_match_icmp { 52 struct flow_dissector_key_icmp *key, *mask; 53 }; 54 55 struct flow_match_tcp { 56 struct flow_dissector_key_tcp *key, *mask; 57 }; 58 59 struct flow_match_mpls { 60 struct flow_dissector_key_mpls *key, *mask; 61 }; 62 63 struct flow_match_enc_keyid { 64 struct flow_dissector_key_keyid *key, *mask; 65 }; 66 67 struct flow_match_enc_opts { 68 struct flow_dissector_key_enc_opts *key, *mask; 69 }; 70 71 struct flow_match_ct { 72 struct flow_dissector_key_ct *key, *mask; 73 }; 74 75 struct flow_rule; 76 77 void flow_rule_match_meta(const struct flow_rule *rule, 78 struct flow_match_meta *out); 79 void flow_rule_match_basic(const struct flow_rule *rule, 80 struct flow_match_basic *out); 81 void flow_rule_match_control(const struct flow_rule *rule, 82 struct flow_match_control *out); 83 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 84 struct flow_match_eth_addrs *out); 85 void flow_rule_match_vlan(const struct flow_rule *rule, 86 struct flow_match_vlan *out); 87 void flow_rule_match_cvlan(const struct flow_rule *rule, 88 struct flow_match_vlan *out); 89 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 90 struct flow_match_ipv4_addrs *out); 91 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 92 struct flow_match_ipv6_addrs *out); 93 void flow_rule_match_ip(const struct flow_rule *rule, 94 struct flow_match_ip *out); 95 void flow_rule_match_ports(const struct flow_rule *rule, 96 struct flow_match_ports *out); 97 void flow_rule_match_tcp(const struct flow_rule *rule, 98 struct flow_match_tcp *out); 99 void flow_rule_match_icmp(const struct flow_rule *rule, 100 struct flow_match_icmp *out); 101 void flow_rule_match_mpls(const struct flow_rule *rule, 102 struct flow_match_mpls *out); 103 void flow_rule_match_enc_control(const struct flow_rule *rule, 104 struct flow_match_control *out); 105 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 106 struct flow_match_ipv4_addrs *out); 107 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 108 struct flow_match_ipv6_addrs *out); 109 void flow_rule_match_enc_ip(const struct flow_rule *rule, 110 struct flow_match_ip *out); 111 void flow_rule_match_enc_ports(const struct flow_rule *rule, 112 struct flow_match_ports *out); 113 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 114 struct flow_match_enc_keyid *out); 115 void flow_rule_match_enc_opts(const struct flow_rule *rule, 116 struct flow_match_enc_opts *out); 117 void flow_rule_match_ct(const struct flow_rule *rule, 118 struct flow_match_ct *out); 119 120 enum flow_action_id { 121 FLOW_ACTION_ACCEPT = 0, 122 FLOW_ACTION_DROP, 123 FLOW_ACTION_TRAP, 124 FLOW_ACTION_GOTO, 125 FLOW_ACTION_REDIRECT, 126 FLOW_ACTION_MIRRED, 127 FLOW_ACTION_REDIRECT_INGRESS, 128 FLOW_ACTION_MIRRED_INGRESS, 129 FLOW_ACTION_VLAN_PUSH, 130 FLOW_ACTION_VLAN_POP, 131 FLOW_ACTION_VLAN_MANGLE, 132 FLOW_ACTION_TUNNEL_ENCAP, 133 FLOW_ACTION_TUNNEL_DECAP, 134 FLOW_ACTION_MANGLE, 135 FLOW_ACTION_ADD, 136 FLOW_ACTION_CSUM, 137 FLOW_ACTION_MARK, 138 FLOW_ACTION_PTYPE, 139 FLOW_ACTION_PRIORITY, 140 FLOW_ACTION_WAKE, 141 FLOW_ACTION_QUEUE, 142 FLOW_ACTION_SAMPLE, 143 FLOW_ACTION_POLICE, 144 FLOW_ACTION_CT, 145 FLOW_ACTION_CT_METADATA, 146 FLOW_ACTION_MPLS_PUSH, 147 FLOW_ACTION_MPLS_POP, 148 FLOW_ACTION_MPLS_MANGLE, 149 FLOW_ACTION_GATE, 150 FLOW_ACTION_PPPOE_PUSH, 151 FLOW_ACTION_JUMP, 152 FLOW_ACTION_PIPE, 153 FLOW_ACTION_VLAN_PUSH_ETH, 154 FLOW_ACTION_VLAN_POP_ETH, 155 FLOW_ACTION_CONTINUE, 156 NUM_FLOW_ACTIONS, 157 }; 158 159 /* This is mirroring enum pedit_header_type definition for easy mapping between 160 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 161 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 162 */ 163 enum flow_action_mangle_base { 164 FLOW_ACT_MANGLE_UNSPEC = 0, 165 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 166 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 167 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 168 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 169 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 170 }; 171 172 enum flow_action_hw_stats_bit { 173 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 174 FLOW_ACTION_HW_STATS_DELAYED_BIT, 175 FLOW_ACTION_HW_STATS_DISABLED_BIT, 176 177 FLOW_ACTION_HW_STATS_NUM_BITS 178 }; 179 180 enum flow_action_hw_stats { 181 FLOW_ACTION_HW_STATS_IMMEDIATE = 182 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 183 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 184 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 185 FLOW_ACTION_HW_STATS_DELAYED, 186 FLOW_ACTION_HW_STATS_DISABLED = 187 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 188 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 189 }; 190 191 typedef void (*action_destr)(void *priv); 192 193 struct flow_action_cookie { 194 u32 cookie_len; 195 u8 cookie[]; 196 }; 197 198 struct flow_action_cookie *flow_action_cookie_create(void *data, 199 unsigned int len, 200 gfp_t gfp); 201 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 202 203 struct flow_action_entry { 204 enum flow_action_id id; 205 u32 hw_index; 206 enum flow_action_hw_stats hw_stats; 207 action_destr destructor; 208 void *destructor_priv; 209 union { 210 u32 chain_index; /* FLOW_ACTION_GOTO */ 211 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 212 struct { /* FLOW_ACTION_VLAN */ 213 u16 vid; 214 __be16 proto; 215 u8 prio; 216 } vlan; 217 struct { /* FLOW_ACTION_VLAN_PUSH_ETH */ 218 unsigned char dst[ETH_ALEN]; 219 unsigned char src[ETH_ALEN]; 220 } vlan_push_eth; 221 struct { /* FLOW_ACTION_MANGLE */ 222 /* FLOW_ACTION_ADD */ 223 enum flow_action_mangle_base htype; 224 u32 offset; 225 u32 mask; 226 u32 val; 227 } mangle; 228 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 229 u32 csum_flags; /* FLOW_ACTION_CSUM */ 230 u32 mark; /* FLOW_ACTION_MARK */ 231 u16 ptype; /* FLOW_ACTION_PTYPE */ 232 u32 priority; /* FLOW_ACTION_PRIORITY */ 233 struct { /* FLOW_ACTION_QUEUE */ 234 u32 ctx; 235 u32 index; 236 u8 vf; 237 } queue; 238 struct { /* FLOW_ACTION_SAMPLE */ 239 struct psample_group *psample_group; 240 u32 rate; 241 u32 trunc_size; 242 bool truncate; 243 } sample; 244 struct { /* FLOW_ACTION_POLICE */ 245 u32 burst; 246 u64 rate_bytes_ps; 247 u64 peakrate_bytes_ps; 248 u32 avrate; 249 u16 overhead; 250 u64 burst_pkt; 251 u64 rate_pkt_ps; 252 u32 mtu; 253 struct { 254 enum flow_action_id act_id; 255 u32 extval; 256 } exceed, notexceed; 257 } police; 258 struct { /* FLOW_ACTION_CT */ 259 int action; 260 u16 zone; 261 struct nf_flowtable *flow_table; 262 } ct; 263 struct { 264 unsigned long cookie; 265 u32 mark; 266 u32 labels[4]; 267 bool orig_dir; 268 } ct_metadata; 269 struct { /* FLOW_ACTION_MPLS_PUSH */ 270 u32 label; 271 __be16 proto; 272 u8 tc; 273 u8 bos; 274 u8 ttl; 275 } mpls_push; 276 struct { /* FLOW_ACTION_MPLS_POP */ 277 __be16 proto; 278 } mpls_pop; 279 struct { /* FLOW_ACTION_MPLS_MANGLE */ 280 u32 label; 281 u8 tc; 282 u8 bos; 283 u8 ttl; 284 } mpls_mangle; 285 struct { 286 s32 prio; 287 u64 basetime; 288 u64 cycletime; 289 u64 cycletimeext; 290 u32 num_entries; 291 struct action_gate_entry *entries; 292 } gate; 293 struct { /* FLOW_ACTION_PPPOE_PUSH */ 294 u16 sid; 295 } pppoe; 296 }; 297 struct flow_action_cookie *cookie; /* user defined action cookie */ 298 }; 299 300 struct flow_action { 301 unsigned int num_entries; 302 struct flow_action_entry entries[]; 303 }; 304 305 static inline bool flow_action_has_entries(const struct flow_action *action) 306 { 307 return action->num_entries; 308 } 309 310 /** 311 * flow_offload_has_one_action() - check if exactly one action is present 312 * @action: tc filter flow offload action 313 * 314 * Returns true if exactly one action is present. 315 */ 316 static inline bool flow_offload_has_one_action(const struct flow_action *action) 317 { 318 return action->num_entries == 1; 319 } 320 321 static inline bool flow_action_is_last_entry(const struct flow_action *action, 322 const struct flow_action_entry *entry) 323 { 324 return entry == &action->entries[action->num_entries - 1]; 325 } 326 327 #define flow_action_for_each(__i, __act, __actions) \ 328 for (__i = 0, __act = &(__actions)->entries[0]; \ 329 __i < (__actions)->num_entries; \ 330 __act = &(__actions)->entries[++__i]) 331 332 static inline bool 333 flow_action_mixed_hw_stats_check(const struct flow_action *action, 334 struct netlink_ext_ack *extack) 335 { 336 const struct flow_action_entry *action_entry; 337 u8 last_hw_stats; 338 int i; 339 340 if (flow_offload_has_one_action(action)) 341 return true; 342 343 flow_action_for_each(i, action_entry, action) { 344 if (i && action_entry->hw_stats != last_hw_stats) { 345 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 346 return false; 347 } 348 last_hw_stats = action_entry->hw_stats; 349 } 350 return true; 351 } 352 353 static inline const struct flow_action_entry * 354 flow_action_first_entry_get(const struct flow_action *action) 355 { 356 WARN_ON(!flow_action_has_entries(action)); 357 return &action->entries[0]; 358 } 359 360 static inline bool 361 __flow_action_hw_stats_check(const struct flow_action *action, 362 struct netlink_ext_ack *extack, 363 bool check_allow_bit, 364 enum flow_action_hw_stats_bit allow_bit) 365 { 366 const struct flow_action_entry *action_entry; 367 368 if (!flow_action_has_entries(action)) 369 return true; 370 if (!flow_action_mixed_hw_stats_check(action, extack)) 371 return false; 372 373 action_entry = flow_action_first_entry_get(action); 374 375 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 376 WARN_ON_ONCE(!action_entry->hw_stats); 377 378 if (!check_allow_bit && 379 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 380 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 381 return false; 382 } else if (check_allow_bit && 383 !(action_entry->hw_stats & BIT(allow_bit))) { 384 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 385 return false; 386 } 387 return true; 388 } 389 390 static inline bool 391 flow_action_hw_stats_check(const struct flow_action *action, 392 struct netlink_ext_ack *extack, 393 enum flow_action_hw_stats_bit allow_bit) 394 { 395 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 396 } 397 398 static inline bool 399 flow_action_basic_hw_stats_check(const struct flow_action *action, 400 struct netlink_ext_ack *extack) 401 { 402 return __flow_action_hw_stats_check(action, extack, false, 0); 403 } 404 405 struct flow_rule { 406 struct flow_match match; 407 struct flow_action action; 408 }; 409 410 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 411 412 static inline bool flow_rule_match_key(const struct flow_rule *rule, 413 enum flow_dissector_key_id key) 414 { 415 return dissector_uses_key(rule->match.dissector, key); 416 } 417 418 struct flow_stats { 419 u64 pkts; 420 u64 bytes; 421 u64 drops; 422 u64 lastused; 423 enum flow_action_hw_stats used_hw_stats; 424 bool used_hw_stats_valid; 425 }; 426 427 static inline void flow_stats_update(struct flow_stats *flow_stats, 428 u64 bytes, u64 pkts, 429 u64 drops, u64 lastused, 430 enum flow_action_hw_stats used_hw_stats) 431 { 432 flow_stats->pkts += pkts; 433 flow_stats->bytes += bytes; 434 flow_stats->drops += drops; 435 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 436 437 /* The driver should pass value with a maximum of one bit set. 438 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 439 */ 440 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 441 flow_stats->used_hw_stats |= used_hw_stats; 442 flow_stats->used_hw_stats_valid = true; 443 } 444 445 enum flow_block_command { 446 FLOW_BLOCK_BIND, 447 FLOW_BLOCK_UNBIND, 448 }; 449 450 enum flow_block_binder_type { 451 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 452 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 453 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 454 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 455 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 456 }; 457 458 struct flow_block { 459 struct list_head cb_list; 460 }; 461 462 struct netlink_ext_ack; 463 464 struct flow_block_offload { 465 enum flow_block_command command; 466 enum flow_block_binder_type binder_type; 467 bool block_shared; 468 bool unlocked_driver_cb; 469 struct net *net; 470 struct flow_block *block; 471 struct list_head cb_list; 472 struct list_head *driver_block_list; 473 struct netlink_ext_ack *extack; 474 struct Qdisc *sch; 475 struct list_head *cb_list_head; 476 }; 477 478 enum tc_setup_type; 479 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 480 void *cb_priv); 481 482 struct flow_block_cb; 483 484 struct flow_block_indr { 485 struct list_head list; 486 struct net_device *dev; 487 struct Qdisc *sch; 488 enum flow_block_binder_type binder_type; 489 void *data; 490 void *cb_priv; 491 void (*cleanup)(struct flow_block_cb *block_cb); 492 }; 493 494 struct flow_block_cb { 495 struct list_head driver_list; 496 struct list_head list; 497 flow_setup_cb_t *cb; 498 void *cb_ident; 499 void *cb_priv; 500 void (*release)(void *cb_priv); 501 struct flow_block_indr indr; 502 unsigned int refcnt; 503 }; 504 505 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 506 void *cb_ident, void *cb_priv, 507 void (*release)(void *cb_priv)); 508 struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 509 void *cb_ident, void *cb_priv, 510 void (*release)(void *cb_priv), 511 struct flow_block_offload *bo, 512 struct net_device *dev, 513 struct Qdisc *sch, void *data, 514 void *indr_cb_priv, 515 void (*cleanup)(struct flow_block_cb *block_cb)); 516 void flow_block_cb_free(struct flow_block_cb *block_cb); 517 518 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 519 flow_setup_cb_t *cb, void *cb_ident); 520 521 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 522 void flow_block_cb_incref(struct flow_block_cb *block_cb); 523 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 524 525 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 526 struct flow_block_offload *offload) 527 { 528 list_add_tail(&block_cb->list, &offload->cb_list); 529 } 530 531 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 532 struct flow_block_offload *offload) 533 { 534 list_move(&block_cb->list, &offload->cb_list); 535 } 536 537 static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 538 struct flow_block_offload *offload) 539 { 540 list_del(&block_cb->indr.list); 541 list_move(&block_cb->list, &offload->cb_list); 542 } 543 544 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 545 struct list_head *driver_block_list); 546 547 int flow_block_cb_setup_simple(struct flow_block_offload *f, 548 struct list_head *driver_list, 549 flow_setup_cb_t *cb, 550 void *cb_ident, void *cb_priv, bool ingress_only); 551 552 enum flow_cls_command { 553 FLOW_CLS_REPLACE, 554 FLOW_CLS_DESTROY, 555 FLOW_CLS_STATS, 556 FLOW_CLS_TMPLT_CREATE, 557 FLOW_CLS_TMPLT_DESTROY, 558 }; 559 560 struct flow_cls_common_offload { 561 u32 chain_index; 562 __be16 protocol; 563 u32 prio; 564 struct netlink_ext_ack *extack; 565 }; 566 567 struct flow_cls_offload { 568 struct flow_cls_common_offload common; 569 enum flow_cls_command command; 570 unsigned long cookie; 571 struct flow_rule *rule; 572 struct flow_stats stats; 573 u32 classid; 574 }; 575 576 enum offload_act_command { 577 FLOW_ACT_REPLACE, 578 FLOW_ACT_DESTROY, 579 FLOW_ACT_STATS, 580 }; 581 582 struct flow_offload_action { 583 struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ 584 enum offload_act_command command; 585 enum flow_action_id id; 586 u32 index; 587 struct flow_stats stats; 588 struct flow_action action; 589 }; 590 591 struct flow_offload_action *offload_action_alloc(unsigned int num_actions); 592 593 static inline struct flow_rule * 594 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 595 { 596 return flow_cmd->rule; 597 } 598 599 static inline void flow_block_init(struct flow_block *flow_block) 600 { 601 INIT_LIST_HEAD(&flow_block->cb_list); 602 } 603 604 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 605 enum tc_setup_type type, void *type_data, 606 void *data, 607 void (*cleanup)(struct flow_block_cb *block_cb)); 608 609 int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 610 void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 611 void (*release)(void *cb_priv)); 612 int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 613 enum tc_setup_type type, void *data, 614 struct flow_block_offload *bo, 615 void (*cleanup)(struct flow_block_cb *block_cb)); 616 bool flow_indr_dev_exists(void); 617 618 #endif /* _NET_FLOW_OFFLOAD_H */ 619