1 #ifndef _NET_FLOW_OFFLOAD_H 2 #define _NET_FLOW_OFFLOAD_H 3 4 #include <linux/kernel.h> 5 #include <linux/list.h> 6 #include <linux/netlink.h> 7 #include <net/flow_dissector.h> 8 9 struct flow_match { 10 struct flow_dissector *dissector; 11 void *mask; 12 void *key; 13 }; 14 15 struct flow_match_meta { 16 struct flow_dissector_key_meta *key, *mask; 17 }; 18 19 struct flow_match_basic { 20 struct flow_dissector_key_basic *key, *mask; 21 }; 22 23 struct flow_match_control { 24 struct flow_dissector_key_control *key, *mask; 25 }; 26 27 struct flow_match_eth_addrs { 28 struct flow_dissector_key_eth_addrs *key, *mask; 29 }; 30 31 struct flow_match_vlan { 32 struct flow_dissector_key_vlan *key, *mask; 33 }; 34 35 struct flow_match_ipv4_addrs { 36 struct flow_dissector_key_ipv4_addrs *key, *mask; 37 }; 38 39 struct flow_match_ipv6_addrs { 40 struct flow_dissector_key_ipv6_addrs *key, *mask; 41 }; 42 43 struct flow_match_ip { 44 struct flow_dissector_key_ip *key, *mask; 45 }; 46 47 struct flow_match_ports { 48 struct flow_dissector_key_ports *key, *mask; 49 }; 50 51 struct flow_match_icmp { 52 struct flow_dissector_key_icmp *key, *mask; 53 }; 54 55 struct flow_match_tcp { 56 struct flow_dissector_key_tcp *key, *mask; 57 }; 58 59 struct flow_match_mpls { 60 struct flow_dissector_key_mpls *key, *mask; 61 }; 62 63 struct flow_match_enc_keyid { 64 struct flow_dissector_key_keyid *key, *mask; 65 }; 66 67 struct flow_match_enc_opts { 68 struct flow_dissector_key_enc_opts *key, *mask; 69 }; 70 71 struct flow_match_ct { 72 struct flow_dissector_key_ct *key, *mask; 73 }; 74 75 struct flow_rule; 76 77 void flow_rule_match_meta(const struct flow_rule *rule, 78 struct flow_match_meta *out); 79 void flow_rule_match_basic(const struct flow_rule *rule, 80 struct flow_match_basic *out); 81 void flow_rule_match_control(const struct flow_rule *rule, 82 struct flow_match_control *out); 83 void flow_rule_match_eth_addrs(const struct flow_rule *rule, 84 struct flow_match_eth_addrs *out); 85 void flow_rule_match_vlan(const struct flow_rule *rule, 86 struct flow_match_vlan *out); 87 void flow_rule_match_cvlan(const struct flow_rule *rule, 88 struct flow_match_vlan *out); 89 void flow_rule_match_ipv4_addrs(const struct flow_rule *rule, 90 struct flow_match_ipv4_addrs *out); 91 void flow_rule_match_ipv6_addrs(const struct flow_rule *rule, 92 struct flow_match_ipv6_addrs *out); 93 void flow_rule_match_ip(const struct flow_rule *rule, 94 struct flow_match_ip *out); 95 void flow_rule_match_ports(const struct flow_rule *rule, 96 struct flow_match_ports *out); 97 void flow_rule_match_tcp(const struct flow_rule *rule, 98 struct flow_match_tcp *out); 99 void flow_rule_match_icmp(const struct flow_rule *rule, 100 struct flow_match_icmp *out); 101 void flow_rule_match_mpls(const struct flow_rule *rule, 102 struct flow_match_mpls *out); 103 void flow_rule_match_enc_control(const struct flow_rule *rule, 104 struct flow_match_control *out); 105 void flow_rule_match_enc_ipv4_addrs(const struct flow_rule *rule, 106 struct flow_match_ipv4_addrs *out); 107 void flow_rule_match_enc_ipv6_addrs(const struct flow_rule *rule, 108 struct flow_match_ipv6_addrs *out); 109 void flow_rule_match_enc_ip(const struct flow_rule *rule, 110 struct flow_match_ip *out); 111 void flow_rule_match_enc_ports(const struct flow_rule *rule, 112 struct flow_match_ports *out); 113 void flow_rule_match_enc_keyid(const struct flow_rule *rule, 114 struct flow_match_enc_keyid *out); 115 void flow_rule_match_enc_opts(const struct flow_rule *rule, 116 struct flow_match_enc_opts *out); 117 void flow_rule_match_ct(const struct flow_rule *rule, 118 struct flow_match_ct *out); 119 120 enum flow_action_id { 121 FLOW_ACTION_ACCEPT = 0, 122 FLOW_ACTION_DROP, 123 FLOW_ACTION_TRAP, 124 FLOW_ACTION_GOTO, 125 FLOW_ACTION_REDIRECT, 126 FLOW_ACTION_MIRRED, 127 FLOW_ACTION_REDIRECT_INGRESS, 128 FLOW_ACTION_MIRRED_INGRESS, 129 FLOW_ACTION_VLAN_PUSH, 130 FLOW_ACTION_VLAN_POP, 131 FLOW_ACTION_VLAN_MANGLE, 132 FLOW_ACTION_TUNNEL_ENCAP, 133 FLOW_ACTION_TUNNEL_DECAP, 134 FLOW_ACTION_MANGLE, 135 FLOW_ACTION_ADD, 136 FLOW_ACTION_CSUM, 137 FLOW_ACTION_MARK, 138 FLOW_ACTION_PTYPE, 139 FLOW_ACTION_PRIORITY, 140 FLOW_ACTION_WAKE, 141 FLOW_ACTION_QUEUE, 142 FLOW_ACTION_SAMPLE, 143 FLOW_ACTION_POLICE, 144 FLOW_ACTION_CT, 145 FLOW_ACTION_CT_METADATA, 146 FLOW_ACTION_MPLS_PUSH, 147 FLOW_ACTION_MPLS_POP, 148 FLOW_ACTION_MPLS_MANGLE, 149 FLOW_ACTION_GATE, 150 FLOW_ACTION_PPPOE_PUSH, 151 NUM_FLOW_ACTIONS, 152 }; 153 154 /* This is mirroring enum pedit_header_type definition for easy mapping between 155 * tc pedit action. Legacy TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK is mapped to 156 * FLOW_ACT_MANGLE_UNSPEC, which is supported by no driver. 157 */ 158 enum flow_action_mangle_base { 159 FLOW_ACT_MANGLE_UNSPEC = 0, 160 FLOW_ACT_MANGLE_HDR_TYPE_ETH, 161 FLOW_ACT_MANGLE_HDR_TYPE_IP4, 162 FLOW_ACT_MANGLE_HDR_TYPE_IP6, 163 FLOW_ACT_MANGLE_HDR_TYPE_TCP, 164 FLOW_ACT_MANGLE_HDR_TYPE_UDP, 165 }; 166 167 enum flow_action_hw_stats_bit { 168 FLOW_ACTION_HW_STATS_IMMEDIATE_BIT, 169 FLOW_ACTION_HW_STATS_DELAYED_BIT, 170 FLOW_ACTION_HW_STATS_DISABLED_BIT, 171 172 FLOW_ACTION_HW_STATS_NUM_BITS 173 }; 174 175 enum flow_action_hw_stats { 176 FLOW_ACTION_HW_STATS_IMMEDIATE = 177 BIT(FLOW_ACTION_HW_STATS_IMMEDIATE_BIT), 178 FLOW_ACTION_HW_STATS_DELAYED = BIT(FLOW_ACTION_HW_STATS_DELAYED_BIT), 179 FLOW_ACTION_HW_STATS_ANY = FLOW_ACTION_HW_STATS_IMMEDIATE | 180 FLOW_ACTION_HW_STATS_DELAYED, 181 FLOW_ACTION_HW_STATS_DISABLED = 182 BIT(FLOW_ACTION_HW_STATS_DISABLED_BIT), 183 FLOW_ACTION_HW_STATS_DONT_CARE = BIT(FLOW_ACTION_HW_STATS_NUM_BITS) - 1, 184 }; 185 186 typedef void (*action_destr)(void *priv); 187 188 struct flow_action_cookie { 189 u32 cookie_len; 190 u8 cookie[]; 191 }; 192 193 struct flow_action_cookie *flow_action_cookie_create(void *data, 194 unsigned int len, 195 gfp_t gfp); 196 void flow_action_cookie_destroy(struct flow_action_cookie *cookie); 197 198 struct flow_action_entry { 199 enum flow_action_id id; 200 enum flow_action_hw_stats hw_stats; 201 action_destr destructor; 202 void *destructor_priv; 203 union { 204 u32 chain_index; /* FLOW_ACTION_GOTO */ 205 struct net_device *dev; /* FLOW_ACTION_REDIRECT */ 206 struct { /* FLOW_ACTION_VLAN */ 207 u16 vid; 208 __be16 proto; 209 u8 prio; 210 } vlan; 211 struct { /* FLOW_ACTION_MANGLE */ 212 /* FLOW_ACTION_ADD */ 213 enum flow_action_mangle_base htype; 214 u32 offset; 215 u32 mask; 216 u32 val; 217 } mangle; 218 struct ip_tunnel_info *tunnel; /* FLOW_ACTION_TUNNEL_ENCAP */ 219 u32 csum_flags; /* FLOW_ACTION_CSUM */ 220 u32 mark; /* FLOW_ACTION_MARK */ 221 u16 ptype; /* FLOW_ACTION_PTYPE */ 222 u32 priority; /* FLOW_ACTION_PRIORITY */ 223 struct { /* FLOW_ACTION_QUEUE */ 224 u32 ctx; 225 u32 index; 226 u8 vf; 227 } queue; 228 struct { /* FLOW_ACTION_SAMPLE */ 229 struct psample_group *psample_group; 230 u32 rate; 231 u32 trunc_size; 232 bool truncate; 233 } sample; 234 struct { /* FLOW_ACTION_POLICE */ 235 u32 index; 236 u32 burst; 237 u64 rate_bytes_ps; 238 u64 burst_pkt; 239 u64 rate_pkt_ps; 240 u32 mtu; 241 } police; 242 struct { /* FLOW_ACTION_CT */ 243 int action; 244 u16 zone; 245 struct nf_flowtable *flow_table; 246 } ct; 247 struct { 248 unsigned long cookie; 249 u32 mark; 250 u32 labels[4]; 251 bool orig_dir; 252 } ct_metadata; 253 struct { /* FLOW_ACTION_MPLS_PUSH */ 254 u32 label; 255 __be16 proto; 256 u8 tc; 257 u8 bos; 258 u8 ttl; 259 } mpls_push; 260 struct { /* FLOW_ACTION_MPLS_POP */ 261 __be16 proto; 262 } mpls_pop; 263 struct { /* FLOW_ACTION_MPLS_MANGLE */ 264 u32 label; 265 u8 tc; 266 u8 bos; 267 u8 ttl; 268 } mpls_mangle; 269 struct { 270 u32 index; 271 s32 prio; 272 u64 basetime; 273 u64 cycletime; 274 u64 cycletimeext; 275 u32 num_entries; 276 struct action_gate_entry *entries; 277 } gate; 278 struct { /* FLOW_ACTION_PPPOE_PUSH */ 279 u16 sid; 280 } pppoe; 281 }; 282 struct flow_action_cookie *cookie; /* user defined action cookie */ 283 }; 284 285 struct flow_action { 286 unsigned int num_entries; 287 struct flow_action_entry entries[]; 288 }; 289 290 static inline bool flow_action_has_entries(const struct flow_action *action) 291 { 292 return action->num_entries; 293 } 294 295 /** 296 * flow_action_has_one_action() - check if exactly one action is present 297 * @action: tc filter flow offload action 298 * 299 * Returns true if exactly one action is present. 300 */ 301 static inline bool flow_offload_has_one_action(const struct flow_action *action) 302 { 303 return action->num_entries == 1; 304 } 305 306 #define flow_action_for_each(__i, __act, __actions) \ 307 for (__i = 0, __act = &(__actions)->entries[0]; \ 308 __i < (__actions)->num_entries; \ 309 __act = &(__actions)->entries[++__i]) 310 311 static inline bool 312 flow_action_mixed_hw_stats_check(const struct flow_action *action, 313 struct netlink_ext_ack *extack) 314 { 315 const struct flow_action_entry *action_entry; 316 u8 last_hw_stats; 317 int i; 318 319 if (flow_offload_has_one_action(action)) 320 return true; 321 322 if (action) { 323 flow_action_for_each(i, action_entry, action) { 324 if (i && action_entry->hw_stats != last_hw_stats) { 325 NL_SET_ERR_MSG_MOD(extack, "Mixing HW stats types for actions is not supported"); 326 return false; 327 } 328 last_hw_stats = action_entry->hw_stats; 329 } 330 } 331 return true; 332 } 333 334 static inline const struct flow_action_entry * 335 flow_action_first_entry_get(const struct flow_action *action) 336 { 337 WARN_ON(!flow_action_has_entries(action)); 338 return &action->entries[0]; 339 } 340 341 static inline bool 342 __flow_action_hw_stats_check(const struct flow_action *action, 343 struct netlink_ext_ack *extack, 344 bool check_allow_bit, 345 enum flow_action_hw_stats_bit allow_bit) 346 { 347 const struct flow_action_entry *action_entry; 348 349 if (!flow_action_has_entries(action)) 350 return true; 351 if (!flow_action_mixed_hw_stats_check(action, extack)) 352 return false; 353 354 action_entry = flow_action_first_entry_get(action); 355 356 /* Zero is not a legal value for hw_stats, catch anyone passing it */ 357 WARN_ON_ONCE(!action_entry->hw_stats); 358 359 if (!check_allow_bit && 360 ~action_entry->hw_stats & FLOW_ACTION_HW_STATS_ANY) { 361 NL_SET_ERR_MSG_MOD(extack, "Driver supports only default HW stats type \"any\""); 362 return false; 363 } else if (check_allow_bit && 364 !(action_entry->hw_stats & BIT(allow_bit))) { 365 NL_SET_ERR_MSG_MOD(extack, "Driver does not support selected HW stats type"); 366 return false; 367 } 368 return true; 369 } 370 371 static inline bool 372 flow_action_hw_stats_check(const struct flow_action *action, 373 struct netlink_ext_ack *extack, 374 enum flow_action_hw_stats_bit allow_bit) 375 { 376 return __flow_action_hw_stats_check(action, extack, true, allow_bit); 377 } 378 379 static inline bool 380 flow_action_basic_hw_stats_check(const struct flow_action *action, 381 struct netlink_ext_ack *extack) 382 { 383 return __flow_action_hw_stats_check(action, extack, false, 0); 384 } 385 386 struct flow_rule { 387 struct flow_match match; 388 struct flow_action action; 389 }; 390 391 struct flow_rule *flow_rule_alloc(unsigned int num_actions); 392 393 static inline bool flow_rule_match_key(const struct flow_rule *rule, 394 enum flow_dissector_key_id key) 395 { 396 return dissector_uses_key(rule->match.dissector, key); 397 } 398 399 struct flow_stats { 400 u64 pkts; 401 u64 bytes; 402 u64 drops; 403 u64 lastused; 404 enum flow_action_hw_stats used_hw_stats; 405 bool used_hw_stats_valid; 406 }; 407 408 static inline void flow_stats_update(struct flow_stats *flow_stats, 409 u64 bytes, u64 pkts, 410 u64 drops, u64 lastused, 411 enum flow_action_hw_stats used_hw_stats) 412 { 413 flow_stats->pkts += pkts; 414 flow_stats->bytes += bytes; 415 flow_stats->drops += drops; 416 flow_stats->lastused = max_t(u64, flow_stats->lastused, lastused); 417 418 /* The driver should pass value with a maximum of one bit set. 419 * Passing FLOW_ACTION_HW_STATS_ANY is invalid. 420 */ 421 WARN_ON(used_hw_stats == FLOW_ACTION_HW_STATS_ANY); 422 flow_stats->used_hw_stats |= used_hw_stats; 423 flow_stats->used_hw_stats_valid = true; 424 } 425 426 enum flow_block_command { 427 FLOW_BLOCK_BIND, 428 FLOW_BLOCK_UNBIND, 429 }; 430 431 enum flow_block_binder_type { 432 FLOW_BLOCK_BINDER_TYPE_UNSPEC, 433 FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 434 FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS, 435 FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP, 436 FLOW_BLOCK_BINDER_TYPE_RED_MARK, 437 }; 438 439 struct flow_block { 440 struct list_head cb_list; 441 }; 442 443 struct netlink_ext_ack; 444 445 struct flow_block_offload { 446 enum flow_block_command command; 447 enum flow_block_binder_type binder_type; 448 bool block_shared; 449 bool unlocked_driver_cb; 450 struct net *net; 451 struct flow_block *block; 452 struct list_head cb_list; 453 struct list_head *driver_block_list; 454 struct netlink_ext_ack *extack; 455 struct Qdisc *sch; 456 }; 457 458 enum tc_setup_type; 459 typedef int flow_setup_cb_t(enum tc_setup_type type, void *type_data, 460 void *cb_priv); 461 462 struct flow_block_cb; 463 464 struct flow_block_indr { 465 struct list_head list; 466 struct net_device *dev; 467 struct Qdisc *sch; 468 enum flow_block_binder_type binder_type; 469 void *data; 470 void *cb_priv; 471 void (*cleanup)(struct flow_block_cb *block_cb); 472 }; 473 474 struct flow_block_cb { 475 struct list_head driver_list; 476 struct list_head list; 477 flow_setup_cb_t *cb; 478 void *cb_ident; 479 void *cb_priv; 480 void (*release)(void *cb_priv); 481 struct flow_block_indr indr; 482 unsigned int refcnt; 483 }; 484 485 struct flow_block_cb *flow_block_cb_alloc(flow_setup_cb_t *cb, 486 void *cb_ident, void *cb_priv, 487 void (*release)(void *cb_priv)); 488 struct flow_block_cb *flow_indr_block_cb_alloc(flow_setup_cb_t *cb, 489 void *cb_ident, void *cb_priv, 490 void (*release)(void *cb_priv), 491 struct flow_block_offload *bo, 492 struct net_device *dev, 493 struct Qdisc *sch, void *data, 494 void *indr_cb_priv, 495 void (*cleanup)(struct flow_block_cb *block_cb)); 496 void flow_block_cb_free(struct flow_block_cb *block_cb); 497 498 struct flow_block_cb *flow_block_cb_lookup(struct flow_block *block, 499 flow_setup_cb_t *cb, void *cb_ident); 500 501 void *flow_block_cb_priv(struct flow_block_cb *block_cb); 502 void flow_block_cb_incref(struct flow_block_cb *block_cb); 503 unsigned int flow_block_cb_decref(struct flow_block_cb *block_cb); 504 505 static inline void flow_block_cb_add(struct flow_block_cb *block_cb, 506 struct flow_block_offload *offload) 507 { 508 list_add_tail(&block_cb->list, &offload->cb_list); 509 } 510 511 static inline void flow_block_cb_remove(struct flow_block_cb *block_cb, 512 struct flow_block_offload *offload) 513 { 514 list_move(&block_cb->list, &offload->cb_list); 515 } 516 517 static inline void flow_indr_block_cb_remove(struct flow_block_cb *block_cb, 518 struct flow_block_offload *offload) 519 { 520 list_del(&block_cb->indr.list); 521 list_move(&block_cb->list, &offload->cb_list); 522 } 523 524 bool flow_block_cb_is_busy(flow_setup_cb_t *cb, void *cb_ident, 525 struct list_head *driver_block_list); 526 527 int flow_block_cb_setup_simple(struct flow_block_offload *f, 528 struct list_head *driver_list, 529 flow_setup_cb_t *cb, 530 void *cb_ident, void *cb_priv, bool ingress_only); 531 532 enum flow_cls_command { 533 FLOW_CLS_REPLACE, 534 FLOW_CLS_DESTROY, 535 FLOW_CLS_STATS, 536 FLOW_CLS_TMPLT_CREATE, 537 FLOW_CLS_TMPLT_DESTROY, 538 }; 539 540 struct flow_cls_common_offload { 541 u32 chain_index; 542 __be16 protocol; 543 u32 prio; 544 struct netlink_ext_ack *extack; 545 }; 546 547 struct flow_cls_offload { 548 struct flow_cls_common_offload common; 549 enum flow_cls_command command; 550 unsigned long cookie; 551 struct flow_rule *rule; 552 struct flow_stats stats; 553 u32 classid; 554 }; 555 556 static inline struct flow_rule * 557 flow_cls_offload_flow_rule(struct flow_cls_offload *flow_cmd) 558 { 559 return flow_cmd->rule; 560 } 561 562 static inline void flow_block_init(struct flow_block *flow_block) 563 { 564 INIT_LIST_HEAD(&flow_block->cb_list); 565 } 566 567 typedef int flow_indr_block_bind_cb_t(struct net_device *dev, struct Qdisc *sch, void *cb_priv, 568 enum tc_setup_type type, void *type_data, 569 void *data, 570 void (*cleanup)(struct flow_block_cb *block_cb)); 571 572 int flow_indr_dev_register(flow_indr_block_bind_cb_t *cb, void *cb_priv); 573 void flow_indr_dev_unregister(flow_indr_block_bind_cb_t *cb, void *cb_priv, 574 void (*release)(void *cb_priv)); 575 int flow_indr_dev_setup_offload(struct net_device *dev, struct Qdisc *sch, 576 enum tc_setup_type type, void *data, 577 struct flow_block_offload *bo, 578 void (*cleanup)(struct flow_block_cb *block_cb)); 579 580 #endif /* _NET_FLOW_OFFLOAD_H */ 581