1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 9 struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13 }; 14 15 struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17 }; 18 19 struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21 }; 22 23 struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25 }; 26 27 struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29 }; 30 31 struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33 }; 34 35 struct flow_match_ipv4_addrs { 36 struct flow_dissector_key_ipv4_addrs *key, *mask; 37 }; 38 39 struct flow_match_ipv6_addrs { 40 struct flow_dissector_key_ipv6_addrs *key, *mask; 41 }; 42 43 struct flow_match_ip { 44 struct flow_dissector_key_ip *key, *mask; 45 }; 46 47 struct flow_match_ports { 48 struct flow_dissector_key_ports *key, *mask; 49 }; 50 51 struct flow_match_icmp { 52 struct flow_dissector_key_icmp *key, *mask; 53 }; 54 55 struct flow_match_tcp { 56 struct flow_dissector_key_tcp *key, *mask; 57 }; 58 59 struct flow_match_mpls { 60 struct flow_dissector_key_mpls *key, *mask; 61 }; 62 63 struct flow_match_enc_keyid { 64 struct flow_dissector_key_keyid *key, *mask; 65 }; 66 67 struct flow_match_enc_opts { 68 struct flow_dissector_key_enc_opts *key, *mask; 69 }; 70 71 struct flow_match_ct { 72 struct flow_dissector_key_ct *key, *mask; 73 }; 74 75 struct flow_rule; 76 77 void flow_rule_match_meta(const struct flow_rule *rule, 78 struct flow_match_meta *out); 79 void flow_rule_match_basic(const struct flow_rule *rule, 80 struct flow_match_basic *out); 81 void flow_rule_match_control(const struct flow_rule *rule, 82 struct flow_match_control *out); 83 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 84 struct flow_match_eth_addrs *out); 85 void flow_rule_match_vlan(const struct flow_rule *rule, 86 struct flow_match_vlan *out); 87 void flow_rule_match_cvlan(const struct flow_rule *rule, 88 struct flow_match_vlan *out); 89 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 90 struct flow_match_ipv4_addrs *out); 91 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 92 struct flow_match_ipv6_addrs *out); 93 void flow_rule_match_ip(const struct flow_rule *rule, 94 struct flow_match_ip *out); 95 void flow_rule_match_ports(const struct flow_rule *rule, 96 struct flow_match_ports *out); 97 void flow_rule_match_tcp(const struct flow_rule *rule, 98 struct flow_match_tcp *out); 99 void flow_rule_match_icmp(const struct flow_rule *rule, 100 struct flow_match_icmp *out); 101 void flow_rule_match_mpls(const struct flow_rule *rule, 102 struct flow_match_mpls *out); 103 void flow_rule_match_enc_control(const struct flow_rule *rule, 104 struct flow_match_control *out); 105 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 106 struct flow_match_ipv4_addrs *out); 107 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 108 struct flow_match_ipv6_addrs *out); 109 void flow_rule_match_enc_ip(const struct flow_rule *rule, 110 struct flow_match_ip *out); 111 void flow_rule_match_enc_ports(const struct flow_rule *rule, 112 struct flow_match_ports *out); 113 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 114 struct flow_match_enc_keyid *out); 115 void flow_rule_match_enc_opts(const struct flow_rule *rule, 116 struct flow_match_enc_opts *out); 117 void flow_rule_match_ct(const struct flow_rule *rule, 118 struct flow_match_ct *out); 119 120 enum flow_action_id { 121 FLOW_ACTION_ACCEPT = 0, 122 FLOW_ACTION_DROP, 123 FLOW_ACTION_TRAP, 124 FLOW_ACTION_GOTO, 125 FLOW_ACTION_REDIRECT, 126 FLOW_ACTION_MIRRED, 127 FLOW_ACTION_REDIRECT_INGRESS, 128 FLOW_ACTION_MIRRED_INGRESS, 129 FLOW_ACTION_VLAN_PUSH, 130 FLOW_ACTION_VLAN_POP, 131 FLOW_ACTION_VLAN_MANGLE, 132 FLOW_ACTION_TUNNEL_ENCAP, 133 FLOW_ACTION_TUNNEL_DECAP, 134 FLOW_ACTION_MANGLE, 135 FLOW_ACTION_ADD, 136 FLOW_ACTION_CSUM, 137 FLOW_ACTION_MARK, 138 FLOW_ACTION_PTYPE, 139 FLOW_ACTION_PRIORITY, 140 FLOW_ACTION_WAKE, 141 FLOW_ACTION_QUEUE, 142 FLOW_ACTION_SAMPLE, 143 FLOW_ACTION_POLICE, 144 FLOW_ACTION_CT, 145 FLOW_ACTION_CT_METADATA, 146 FLOW_ACTION_MPLS_PUSH, 147 FLOW_ACTION_MPLS_POP, 148 FLOW_ACTION_MPLS_MANGLE, 149 FLOW_ACTION_GATE, 150 FLOW_ACTION_PPPOE_PUSH, 151 NUM_FLOW_ACTIONS, 152 }; 153 154 /* This is mirroring enum pedit_header_type definition for easy mapping between 155 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 156 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 157 */ 158 enum flow_action_mangle_base { 159 FLOW_ACT_MANGLE_UNSPEC = 0, 160 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 161 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 162 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 163 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 164 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 165 }; 166 167 enum flow_action_hw_stats_bit { 168 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 169 FLOW_ACTION_HW_STATS_DELAYED_BIT, 170 FLOW_ACTION_HW_STATS_DISABLED_BIT, 171 172 FLOW_ACTION_HW_STATS_NUM_BITS 173 }; 174 175 enum flow_action_hw_stats { 176 FLOW_ACTION_HW_STATS_IMMEDIATE = 177 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 178 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 179 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 180 FLOW_ACTION_HW_STATS_DELAYED, 181 FLOW_ACTION_HW_STATS_DISABLED = 182 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 183 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 184 }; 185 186 typedef void (*action_destr)(void *priv); 187 188 struct flow_action_cookie { 189 u32 cookie_len; 190 u8 cookie[]; 191 }; 192 193 struct flow_action_cookie *flow_action_cookie_create(void *data, 194 unsigned int len, 195 gfp_t gfp); 196 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 197 198 struct flow_action_entry { 199 enum flow_action_id id; 200 u32 hw_index; 201 enum flow_action_hw_stats hw_stats; 202 action_destr destructor; 203 void *destructor_priv; 204 union { 205 u32 chain_index; /* FLOW_ACTION_GOTO */ 206 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 207 struct { /* FLOW_ACTION_VLAN */ 208 u16 vid; 209 __be16 proto; 210 u8 prio; 211 } vlan; 212 struct { /* FLOW_ACTION_MANGLE */ 213 /* FLOW_ACTION_ADD */ 214 enum flow_action_mangle_base htype; 215 u32 offset; 216 u32 mask; 217 u32 val; 218 } mangle; 219 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 220 u32 csum_flags; /* FLOW_ACTION_CSUM */ 221 u32 mark; /* FLOW_ACTION_MARK */ 222 u16 ptype; /* FLOW_ACTION_PTYPE */ 223 u32 priority; /* FLOW_ACTION_PRIORITY */ 224 struct { /* FLOW_ACTION_QUEUE */ 225 u32 ctx; 226 u32 index; 227 u8 vf; 228 } queue; 229 struct { /* FLOW_ACTION_SAMPLE */ 230 struct psample_group *psample_group; 231 u32 rate; 232 u32 trunc_size; 233 bool truncate; 234 } sample; 235 struct { /* FLOW_ACTION_POLICE */ 236 u32 burst; 237 u64 rate_bytes_ps; 238 u64 burst_pkt; 239 u64 rate_pkt_ps; 240 u32 mtu; 241 } police; 242 struct { /* FLOW_ACTION_CT */ 243 int action; 244 u16 zone; 245 struct nf_flowtable *flow_table; 246 } ct; 247 struct { 248 unsigned long cookie; 249 u32 mark; 250 u32 labels[4]; 251 bool orig_dir; 252 } ct_metadata; 253 struct { /* FLOW_ACTION_MPLS_PUSH */ 254 u32 label; 255 __be16 proto; 256 u8 tc; 257 u8 bos; 258 u8 ttl; 259 } mpls_push; 260 struct { /* FLOW_ACTION_MPLS_POP */ 261 __be16 proto; 262 } mpls_pop; 263 struct { /* FLOW_ACTION_MPLS_MANGLE */ 264 u32 label; 265 u8 tc; 266 u8 bos; 267 u8 ttl; 268 } mpls_mangle; 269 struct { 270 s32 prio; 271 u64 basetime; 272 u64 cycletime; 273 u64 cycletimeext; 274 u32 num_entries; 275 struct action_gate_entry *entries; 276 } gate; 277 struct { /* FLOW_ACTION_PPPOE_PUSH */ 278 u16 sid; 279 } pppoe; 280 }; 281 struct flow_action_cookie *cookie; /* user defined action cookie */ 282 }; 283 284 struct flow_action { 285 unsigned int num_entries; 286 struct flow_action_entry entries[]; 287 }; 288 289 static inline bool flow_action_has_entries(const struct flow_action *action) 290 { 291 return action->num_entries; 292 } 293 294 /** 295 * flow_offload_has_one_action() - check if exactly one action is present 296 * @action: tc filter flow offload action 297 * 298 * Returns true if exactly one action is present. 299 */ 300 static inline bool flow_offload_has_one_action(const struct flow_action *action) 301 { 302 return action->num_entries == 1; 303 } 304 305 #define flow_action_for_each(__i, __act, __actions) \ 306 for (__i = 0, __act = &(__actions)->entries[0]; \ 307 __i < (__actions)->num_entries; \ 308 __act = &(__actions)->entries[++__i]) 309 310 static inline bool 311 flow_action_mixed_hw_stats_check(const struct flow_action *action, 312 struct netlink_ext_ack *extack) 313 { 314 const struct flow_action_entry *action_entry; 315 u8 last_hw_stats; 316 int i; 317 318 if (flow_offload_has_one_action(action)) 319 return true; 320 321 flow_action_for_each(i, action_entry, action) { 322 if (i && action_entry->hw_stats != last_hw_stats) { 323 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 324 return false; 325 } 326 last_hw_stats = action_entry->hw_stats; 327 } 328 return true; 329 } 330 331 static inline const struct flow_action_entry * 332 flow_action_first_entry_get(const struct flow_action *action) 333 { 334 WARN_ON(!flow_action_has_entries(action)); 335 return &action->entries[0]; 336 } 337 338 static inline bool 339 __flow_action_hw_stats_check(const struct flow_action *action, 340 struct netlink_ext_ack *extack, 341 bool check_allow_bit, 342 enum flow_action_hw_stats_bit allow_bit) 343 { 344 const struct flow_action_entry *action_entry; 345 346 if (!flow_action_has_entries(action)) 347 return true; 348 if (!flow_action_mixed_hw_stats_check(action, extack)) 349 return false; 350 351 action_entry = flow_action_first_entry_get(action); 352 353 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 354 WARN_ON_ONCE(!action_entry->hw_stats); 355 356 if (!check_allow_bit && 357 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 358 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 359 return false; 360 } else if (check_allow_bit && 361 !(action_entry->hw_stats & BIT(allow_bit))) { 362 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 363 return false; 364 } 365 return true; 366 } 367 368 static inline bool 369 flow_action_hw_stats_check(const struct flow_action *action, 370 struct netlink_ext_ack *extack, 371 enum flow_action_hw_stats_bit allow_bit) 372 { 373 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 374 } 375 376 static inline bool 377 flow_action_basic_hw_stats_check(const struct flow_action *action, 378 struct netlink_ext_ack *extack) 379 { 380 return __flow_action_hw_stats_check(action, extack, false, 0); 381 } 382 383 struct flow_rule { 384 struct flow_match match; 385 struct flow_action action; 386 }; 387 388 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 389 390 static inline bool flow_rule_match_key(const struct flow_rule *rule, 391 enum flow_dissector_key_id key) 392 { 393 return dissector_uses_key(rule->match.dissector, key); 394 } 395 396 struct flow_stats { 397 u64 pkts; 398 u64 bytes; 399 u64 drops; 400 u64 lastused; 401 enum flow_action_hw_stats used_hw_stats; 402 bool used_hw_stats_valid; 403 }; 404 405 static inline void flow_stats_update(struct flow_stats *flow_stats, 406 u64 bytes, u64 pkts, 407 u64 drops, u64 lastused, 408 enum flow_action_hw_stats used_hw_stats) 409 { 410 flow_stats->pkts += pkts; 411 flow_stats->bytes += bytes; 412 flow_stats->drops += drops; 413 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 414 415 /* The driver should pass value with a maximum of one bit set. 416 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 417 */ 418 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 419 flow_stats->used_hw_stats |= used_hw_stats; 420 flow_stats->used_hw_stats_valid = true; 421 } 422 423 enum flow_block_command { 424 FLOW_BLOCK_BIND, 425 FLOW_BLOCK_UNBIND, 426 }; 427 428 enum flow_block_binder_type { 429 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 430 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 431 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 432 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 433 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 434 }; 435 436 struct flow_block { 437 struct list_head cb_list; 438 }; 439 440 struct netlink_ext_ack; 441 442 struct flow_block_offload { 443 enum flow_block_command command; 444 enum flow_block_binder_type binder_type; 445 bool block_shared; 446 bool unlocked_driver_cb; 447 struct net *net; 448 struct flow_block *block; 449 struct list_head cb_list; 450 struct list_head *driver_block_list; 451 struct netlink_ext_ack *extack; 452 struct Qdisc *sch; 453 struct list_head *cb_list_head; 454 }; 455 456 enum tc_setup_type; 457 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 458 void *cb_priv); 459 460 struct flow_block_cb; 461 462 struct flow_block_indr { 463 struct list_head list; 464 struct net_device *dev; 465 struct Qdisc *sch; 466 enum flow_block_binder_type binder_type; 467 void *data; 468 void *cb_priv; 469 void (*cleanup)(struct flow_block_cb *block_cb); 470 }; 471 472 struct flow_block_cb { 473 struct list_head driver_list; 474 struct list_head list; 475 flow_setup_cb_t *cb; 476 void *cb_ident; 477 void *cb_priv; 478 void (*release)(void *cb_priv); 479 struct flow_block_indr indr; 480 unsigned int refcnt; 481 }; 482 483 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 484 void *cb_ident, void *cb_priv, 485 void (*release)(void *cb_priv)); 486 struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 487 void *cb_ident, void *cb_priv, 488 void (*release)(void *cb_priv), 489 struct flow_block_offload *bo, 490 struct net_device *dev, 491 struct Qdisc *sch, void *data, 492 void *indr_cb_priv, 493 void (*cleanup)(struct flow_block_cb *block_cb)); 494 void flow_block_cb_free(struct flow_block_cb *block_cb); 495 496 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 497 flow_setup_cb_t *cb, void *cb_ident); 498 499 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 500 void flow_block_cb_incref(struct flow_block_cb *block_cb); 501 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 502 503 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 504 struct flow_block_offload *offload) 505 { 506 list_add_tail(&block_cb->list, &offload->cb_list); 507 } 508 509 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 510 struct flow_block_offload *offload) 511 { 512 list_move(&block_cb->list, &offload->cb_list); 513 } 514 515 static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 516 struct flow_block_offload *offload) 517 { 518 list_del(&block_cb->indr.list); 519 list_move(&block_cb->list, &offload->cb_list); 520 } 521 522 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 523 struct list_head *driver_block_list); 524 525 int flow_block_cb_setup_simple(struct flow_block_offload *f, 526 struct list_head *driver_list, 527 flow_setup_cb_t *cb, 528 void *cb_ident, void *cb_priv, bool ingress_only); 529 530 enum flow_cls_command { 531 FLOW_CLS_REPLACE, 532 FLOW_CLS_DESTROY, 533 FLOW_CLS_STATS, 534 FLOW_CLS_TMPLT_CREATE, 535 FLOW_CLS_TMPLT_DESTROY, 536 }; 537 538 struct flow_cls_common_offload { 539 u32 chain_index; 540 __be16 protocol; 541 u32 prio; 542 struct netlink_ext_ack *extack; 543 }; 544 545 struct flow_cls_offload { 546 struct flow_cls_common_offload common; 547 enum flow_cls_command command; 548 unsigned long cookie; 549 struct flow_rule *rule; 550 struct flow_stats stats; 551 u32 classid; 552 }; 553 554 enum offload_act_command { 555 FLOW_ACT_REPLACE, 556 FLOW_ACT_DESTROY, 557 FLOW_ACT_STATS, 558 }; 559 560 struct flow_offload_action { 561 struct netlink_ext_ack *extack; /* NULL in FLOW_ACT_STATS process*/ 562 enum offload_act_command command; 563 enum flow_action_id id; 564 u32 index; 565 struct flow_stats stats; 566 struct flow_action action; 567 }; 568 569 struct flow_offload_action *offload_action_alloc(unsigned int num_actions); 570 571 static inline struct flow_rule * 572 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 573 { 574 return flow_cmd->rule; 575 } 576 577 static inline void flow_block_init(struct flow_block *flow_block) 578 { 579 INIT_LIST_HEAD(&flow_block->cb_list); 580 } 581 582 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 583 enum tc_setup_type type, void *type_data, 584 void *data, 585 void (*cleanup)(struct flow_block_cb *block_cb)); 586 587 int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 588 void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 589 void (*release)(void *cb_priv)); 590 int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 591 enum tc_setup_type type, void *data, 592 struct flow_block_offload *bo, 593 void (*cleanup)(struct flow_block_cb *block_cb)); 594 595 #endif /* _NET_FLOW_OFFLOAD_H */ 596