1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 9 struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13 }; 14 15 struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17 }; 18 19 struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21 }; 22 23 struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25 }; 26 27 struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29 }; 30 31 struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33 }; 34 35 struct flow_match_ipv4_addrs { 36 struct flow_dissector_key_ipv4_addrs *key, *mask; 37 }; 38 39 struct flow_match_ipv6_addrs { 40 struct flow_dissector_key_ipv6_addrs *key, *mask; 41 }; 42 43 struct flow_match_ip { 44 struct flow_dissector_key_ip *key, *mask; 45 }; 46 47 struct flow_match_ports { 48 struct flow_dissector_key_ports *key, *mask; 49 }; 50 51 struct flow_match_icmp { 52 struct flow_dissector_key_icmp *key, *mask; 53 }; 54 55 struct flow_match_tcp { 56 struct flow_dissector_key_tcp *key, *mask; 57 }; 58 59 struct flow_match_mpls { 60 struct flow_dissector_key_mpls *key, *mask; 61 }; 62 63 struct flow_match_enc_keyid { 64 struct flow_dissector_key_keyid *key, *mask; 65 }; 66 67 struct flow_match_enc_opts { 68 struct flow_dissector_key_enc_opts *key, *mask; 69 }; 70 71 struct flow_match_ct { 72 struct flow_dissector_key_ct *key, *mask; 73 }; 74 75 struct flow_rule; 76 77 void flow_rule_match_meta(const struct flow_rule *rule, 78 struct flow_match_meta *out); 79 void flow_rule_match_basic(const struct flow_rule *rule, 80 struct flow_match_basic *out); 81 void flow_rule_match_control(const struct flow_rule *rule, 82 struct flow_match_control *out); 83 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 84 struct flow_match_eth_addrs *out); 85 void flow_rule_match_vlan(const struct flow_rule *rule, 86 struct flow_match_vlan *out); 87 void flow_rule_match_cvlan(const struct flow_rule *rule, 88 struct flow_match_vlan *out); 89 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 90 struct flow_match_ipv4_addrs *out); 91 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 92 struct flow_match_ipv6_addrs *out); 93 void flow_rule_match_ip(const struct flow_rule *rule, 94 struct flow_match_ip *out); 95 void flow_rule_match_ports(const struct flow_rule *rule, 96 struct flow_match_ports *out); 97 void flow_rule_match_tcp(const struct flow_rule *rule, 98 struct flow_match_tcp *out); 99 void flow_rule_match_icmp(const struct flow_rule *rule, 100 struct flow_match_icmp *out); 101 void flow_rule_match_mpls(const struct flow_rule *rule, 102 struct flow_match_mpls *out); 103 void flow_rule_match_enc_control(const struct flow_rule *rule, 104 struct flow_match_control *out); 105 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 106 struct flow_match_ipv4_addrs *out); 107 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 108 struct flow_match_ipv6_addrs *out); 109 void flow_rule_match_enc_ip(const struct flow_rule *rule, 110 struct flow_match_ip *out); 111 void flow_rule_match_enc_ports(const struct flow_rule *rule, 112 struct flow_match_ports *out); 113 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 114 struct flow_match_enc_keyid *out); 115 void flow_rule_match_enc_opts(const struct flow_rule *rule, 116 struct flow_match_enc_opts *out); 117 void flow_rule_match_ct(const struct flow_rule *rule, 118 struct flow_match_ct *out); 119 120 enum flow_action_id { 121 FLOW_ACTION_ACCEPT = 0, 122 FLOW_ACTION_DROP, 123 FLOW_ACTION_TRAP, 124 FLOW_ACTION_GOTO, 125 FLOW_ACTION_REDIRECT, 126 FLOW_ACTION_MIRRED, 127 FLOW_ACTION_REDIRECT_INGRESS, 128 FLOW_ACTION_MIRRED_INGRESS, 129 FLOW_ACTION_VLAN_PUSH, 130 FLOW_ACTION_VLAN_POP, 131 FLOW_ACTION_VLAN_MANGLE, 132 FLOW_ACTION_TUNNEL_ENCAP, 133 FLOW_ACTION_TUNNEL_DECAP, 134 FLOW_ACTION_MANGLE, 135 FLOW_ACTION_ADD, 136 FLOW_ACTION_CSUM, 137 FLOW_ACTION_MARK, 138 FLOW_ACTION_PTYPE, 139 FLOW_ACTION_PRIORITY, 140 FLOW_ACTION_WAKE, 141 FLOW_ACTION_QUEUE, 142 FLOW_ACTION_SAMPLE, 143 FLOW_ACTION_POLICE, 144 FLOW_ACTION_CT, 145 FLOW_ACTION_CT_METADATA, 146 FLOW_ACTION_MPLS_PUSH, 147 FLOW_ACTION_MPLS_POP, 148 FLOW_ACTION_MPLS_MANGLE, 149 FLOW_ACTION_GATE, 150 FLOW_ACTION_PPPOE_PUSH, 151 FLOW_ACTION_JUMP, 152 FLOW_ACTION_PIPE, 153 FLOW_ACTION_VLAN_PUSH_ETH, 154 FLOW_ACTION_VLAN_POP_ETH, 155 NUM_FLOW_ACTIONS, 156 }; 157 158 /* This is mirroring enum pedit_header_type definition for easy mapping between 159 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 160 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 161 */ 162 enum flow_action_mangle_base { 163 FLOW_ACT_MANGLE_UNSPEC = 0, 164 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 165 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 166 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 167 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 168 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 169 }; 170 171 enum flow_action_hw_stats_bit { 172 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 173 FLOW_ACTION_HW_STATS_DELAYED_BIT, 174 FLOW_ACTION_HW_STATS_DISABLED_BIT, 175 176 FLOW_ACTION_HW_STATS_NUM_BITS 177 }; 178 179 enum flow_action_hw_stats { 180 FLOW_ACTION_HW_STATS_IMMEDIATE = 181 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 182 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 183 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 184 FLOW_ACTION_HW_STATS_DELAYED, 185 FLOW_ACTION_HW_STATS_DISABLED = 186 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 187 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 188 }; 189 190 typedef void (*action_destr)(void *priv); 191 192 struct flow_action_cookie { 193 u32 cookie_len; 194 u8 cookie[]; 195 }; 196 197 struct flow_action_cookie *flow_action_cookie_create(void *data, 198 unsigned int len, 199 gfp_t gfp); 200 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 201 202 struct flow_action_entry { 203 enum flow_action_id id; 204 u32 hw_index; 205 enum flow_action_hw_stats hw_stats; 206 action_destr destructor; 207 void *destructor_priv; 208 union { 209 u32 chain_index; /* FLOW_ACTION_GOTO */ 210 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 211 struct { /* FLOW_ACTION_VLAN */ 212 u16 vid; 213 __be16 proto; 214 u8 prio; 215 } vlan; 216 struct { /* FLOW_ACTION_VLAN_PUSH_ETH */ 217 unsigned char dst[ETH_ALEN]; 218 unsigned char src[ETH_ALEN]; 219 } vlan_push_eth; 220 struct { /* FLOW_ACTION_MANGLE */ 221 /* FLOW_ACTION_ADD */ 222 enum flow_action_mangle_base htype; 223 u32 offset; 224 u32 mask; 225 u32 val; 226 } mangle; 227 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 228 u32 csum_flags; /* FLOW_ACTION_CSUM */ 229 u32 mark; /* FLOW_ACTION_MARK */ 230 u16 ptype; /* FLOW_ACTION_PTYPE */ 231 u32 priority; /* FLOW_ACTION_PRIORITY */ 232 struct { /* FLOW_ACTION_QUEUE */ 233 u32 ctx; 234 u32 index; 235 u8 vf; 236 } queue; 237 struct { /* FLOW_ACTION_SAMPLE */ 238 struct psample_group *psample_group; 239 u32 rate; 240 u32 trunc_size; 241 bool truncate; 242 } sample; 243 struct { /* FLOW_ACTION_POLICE */ 244 u32 burst; 245 u64 rate_bytes_ps; 246 u64 peakrate_bytes_ps; 247 u32 avrate; 248 u16 overhead; 249 u64 burst_pkt; 250 u64 rate_pkt_ps; 251 u32 mtu; 252 struct { 253 enum flow_action_id act_id; 254 u32 extval; 255 } exceed, notexceed; 256 } police; 257 struct { /* FLOW_ACTION_CT */ 258 int action; 259 u16 zone; 260 struct nf_flowtable *flow_table; 261 } ct; 262 struct { 263 unsigned long cookie; 264 u32 mark; 265 u32 labels[4]; 266 bool orig_dir; 267 } ct_metadata; 268 struct { /* FLOW_ACTION_MPLS_PUSH */ 269 u32 label; 270 __be16 proto; 271 u8 tc; 272 u8 bos; 273 u8 ttl; 274 } mpls_push; 275 struct { /* FLOW_ACTION_MPLS_POP */ 276 __be16 proto; 277 } mpls_pop; 278 struct { /* FLOW_ACTION_MPLS_MANGLE */ 279 u32 label; 280 u8 tc; 281 u8 bos; 282 u8 ttl; 283 } mpls_mangle; 284 struct { 285 s32 prio; 286 u64 basetime; 287 u64 cycletime; 288 u64 cycletimeext; 289 u32 num_entries; 290 struct action_gate_entry *entries; 291 } gate; 292 struct { /* FLOW_ACTION_PPPOE_PUSH */ 293 u16 sid; 294 } pppoe; 295 }; 296 struct flow_action_cookie *cookie; /* user defined action cookie */ 297 }; 298 299 struct flow_action { 300 unsigned int num_entries; 301 struct flow_action_entry entries[]; 302 }; 303 304 static inline bool flow_action_has_entries(const struct flow_action *action) 305 { 306 return action->num_entries; 307 } 308 309 /** 310 * flow_offload_has_one_action() - check if exactly one action is present 311 * @action: tc filter flow offload action 312 * 313 * Returns true if exactly one action is present. 314 */ 315 static inline bool flow_offload_has_one_action(const struct flow_action *action) 316 { 317 return action->num_entries == 1; 318 } 319 320 static inline bool flow_action_is_last_entry(const struct flow_action *action, 321 const struct flow_action_entry *entry) 322 { 323 return entry == &action->entries[action->num_entries - 1]; 324 } 325 326 #define flow_action_for_each(__i, __act, __actions) \ 327 for (__i = 0, __act = &(__actions)->entries[0]; \ 328 __i < (__actions)->num_entries; \ 329 __act = &(__actions)->entries[++__i]) 330 331 static inline bool 332 flow_action_mixed_hw_stats_check(const struct flow_action *action, 333 struct netlink_ext_ack *extack) 334 { 335 const struct flow_action_entry *action_entry; 336 u8 last_hw_stats; 337 int i; 338 339 if (flow_offload_has_one_action(action)) 340 return true; 341 342 flow_action_for_each(i, action_entry, action) { 343 if (i && action_entry->hw_stats != last_hw_stats) { 344 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 345 return false; 346 } 347 last_hw_stats = action_entry->hw_stats; 348 } 349 return true; 350 } 351 352 static inline const struct flow_action_entry * 353 flow_action_first_entry_get(const struct flow_action *action) 354 { 355 WARN_ON(!flow_action_has_entries(action)); 356 return &action->entries[0]; 357 } 358 359 static inline bool 360 __flow_action_hw_stats_check(const struct flow_action *action, 361 struct netlink_ext_ack *extack, 362 bool check_allow_bit, 363 enum flow_action_hw_stats_bit allow_bit) 364 { 365 const struct flow_action_entry *action_entry; 366 367 if (!flow_action_has_entries(action)) 368 return true; 369 if (!flow_action_mixed_hw_stats_check(action, extack)) 370 return false; 371 372 action_entry = flow_action_first_entry_get(action); 373 374 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 375 WARN_ON_ONCE(!action_entry->hw_stats); 376 377 if (!check_allow_bit && 378 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 379 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 380 return false; 381 } else if (check_allow_bit && 382 !(action_entry->hw_stats & BIT(allow_bit))) { 383 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 384 return false; 385 } 386 return true; 387 } 388 389 static inline bool 390 flow_action_hw_stats_check(const struct flow_action *action, 391 struct netlink_ext_ack *extack, 392 enum flow_action_hw_stats_bit allow_bit) 393 { 394 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 395 } 396 397 static inline bool 398 flow_action_basic_hw_stats_check(const struct flow_action *action, 399 struct netlink_ext_ack *extack) 400 { 401 return __flow_action_hw_stats_check(action, extack, false, 0); 402 } 403 404 struct flow_rule { 405 struct flow_match match; 406 struct flow_action action; 407 }; 408 409 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 410 411 static inline bool flow_rule_match_key(const struct flow_rule *rule, 412 enum flow_dissector_key_id key) 413 { 414 return dissector_uses_key(rule->match.dissector, key); 415 } 416 417 struct flow_stats { 418 u64 pkts; 419 u64 bytes; 420 u64 drops; 421 u64 lastused; 422 enum flow_action_hw_stats used_hw_stats; 423 bool used_hw_stats_valid; 424 }; 425 426 static inline void flow_stats_update(struct flow_stats *flow_stats, 427 u64 bytes, u64 pkts, 428 u64 drops, u64 lastused, 429 enum flow_action_hw_stats used_hw_stats) 430 { 431 flow_stats->pkts += pkts; 432 flow_stats->bytes += bytes; 433 flow_stats->drops += drops; 434 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 435 436 /* The driver should pass value with a maximum of one bit set. 437 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 438 */ 439 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 440 flow_stats->used_hw_stats |= used_hw_stats; 441 flow_stats->used_hw_stats_valid = true; 442 } 443 444 enum flow_block_command { 445 FLOW_BLOCK_BIND, 446 FLOW_BLOCK_UNBIND, 447 }; 448 449 enum flow_block_binder_type { 450 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 451 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 452 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 453 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 454 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 455 }; 456 457 struct flow_block { 458 struct list_head cb_list; 459 }; 460 461 struct netlink_ext_ack; 462 463 struct flow_block_offload { 464 enum flow_block_command command; 465 enum flow_block_binder_type binder_type; 466 bool block_shared; 467 bool unlocked_driver_cb; 468 struct net *net; 469 struct flow_block *block; 470 struct list_head cb_list; 471 struct list_head *driver_block_list; 472 struct netlink_ext_ack *extack; 473 struct Qdisc *sch; 474 struct list_head *cb_list_head; 475 }; 476 477 enum tc_setup_type; 478 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 479 void *cb_priv); 480 481 struct flow_block_cb; 482 483 struct flow_block_indr { 484 struct list_head list; 485 struct net_device *dev; 486 struct Qdisc *sch; 487 enum flow_block_binder_type binder_type; 488 void *data; 489 void *cb_priv; 490 void (*cleanup)(struct flow_block_cb *block_cb); 491 }; 492 493 struct flow_block_cb { 494 struct list_head driver_list; 495 struct list_head list; 496 flow_setup_cb_t *cb; 497 void *cb_ident; 498 void *cb_priv; 499 void (*release)(void *cb_priv); 500 struct flow_block_indr indr; 501 unsigned int refcnt; 502 }; 503 504 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 505 void *cb_ident, void *cb_priv, 506 void (*release)(void *cb_priv)); 507 struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 508 void *cb_ident, void *cb_priv, 509 void (*release)(void *cb_priv), 510 struct flow_block_offload *bo, 511 struct net_device *dev, 512 struct Qdisc *sch, void *data, 513 void *indr_cb_priv, 514 void (*cleanup)(struct flow_block_cb *block_cb)); 515 void flow_block_cb_free(struct flow_block_cb *block_cb); 516 517 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 518 flow_setup_cb_t *cb, void *cb_ident); 519 520 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 521 void flow_block_cb_incref(struct flow_block_cb *block_cb); 522 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 523 524 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 525 struct flow_block_offload *offload) 526 { 527 list_add_tail(&block_cb->list, &offload->cb_list); 528 } 529 530 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 531 struct flow_block_offload *offload) 532 { 533 list_move(&block_cb->list, &offload->cb_list); 534 } 535 536 static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 537 struct flow_block_offload *offload) 538 { 539 list_del(&block_cb->indr.list); 540 list_move(&block_cb->list, &offload->cb_list); 541 } 542 543 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 544 struct list_head *driver_block_list); 545 546 int flow_block_cb_setup_simple(struct flow_block_offload *f, 547 struct list_head *driver_list, 548 flow_setup_cb_t *cb, 549 void *cb_ident, void *cb_priv, bool ingress_only); 550 551 enum flow_cls_command { 552 FLOW_CLS_REPLACE, 553 FLOW_CLS_DESTROY, 554 FLOW_CLS_STATS, 555 FLOW_CLS_TMPLT_CREATE, 556 FLOW_CLS_TMPLT_DESTROY, 557 }; 558 559 struct flow_cls_common_offload { 560 u32 chain_index; 561 __be16 protocol; 562 u32 prio; 563 struct netlink_ext_ack *extack; 564 }; 565 566 struct flow_cls_offload { 567 struct flow_cls_common_offload common; 568 enum flow_cls_command command; 569 unsigned long cookie; 570 struct flow_rule *rule; 571 struct flow_stats stats; 572 u32 classid; 573 }; 574 575 enum offload_act_command { 576 FLOW_ACT_REPLACE, 577 FLOW_ACT_DESTROY, 578 FLOW_ACT_STATS, 579 }; 580 581 struct flow_offload_action { 582 struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ 583 enum offload_act_command command; 584 enum flow_action_id id; 585 u32 index; 586 struct flow_stats stats; 587 struct flow_action action; 588 }; 589 590 struct flow_offload_action *offload_action_alloc(unsigned int num_actions); 591 592 static inline struct flow_rule * 593 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 594 { 595 return flow_cmd->rule; 596 } 597 598 static inline void flow_block_init(struct flow_block *flow_block) 599 { 600 INIT_LIST_HEAD(&flow_block->cb_list); 601 } 602 603 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 604 enum tc_setup_type type, void *type_data, 605 void *data, 606 void (*cleanup)(struct flow_block_cb *block_cb)); 607 608 int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 609 void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 610 void (*release)(void *cb_priv)); 611 int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 612 enum tc_setup_type type, void *data, 613 struct flow_block_offload *bo, 614 void (*cleanup)(struct flow_block_cb *block_cb)); 615 bool flow_indr_dev_exists(void); 616 617 #endif /* _NET_FLOW_OFFLOAD_H */ 618