1 #include <linux/kernel.h> 2 #include <linux/init.h> 3 #include <linux/module.h> 4 #include <linux/netfilter.h> 5 #include <linux/rhashtable.h> 6 #include <linux/netdevice.h> 7 #include <linux/tc_act/tc_csum.h> 8 #include <net/flow_offload.h> 9 #include <net/netfilter/nf_flow_table.h> 10 #include <net/netfilter/nf_tables.h> 11 #include <net/netfilter/nf_conntrack.h> 12 #include <net/netfilter/nf_conntrack_acct.h> 13 #include <net/netfilter/nf_conntrack_core.h> 14 #include <net/netfilter/nf_conntrack_tuple.h> 15 16 static struct workqueue_struct *nf_flow_offload_add_wq; 17 static struct workqueue_struct *nf_flow_offload_del_wq; 18 static struct workqueue_struct *nf_flow_offload_stats_wq; 19 20 struct flow_offload_work { 21 struct list_head list; 22 enum flow_cls_command cmd; 23 struct nf_flowtable *flowtable; 24 struct flow_offload *flow; 25 struct work_struct work; 26 }; 27 28 #define NF_FLOW_DISSECTOR(__match, __type, __field) \ 29 (__match)->dissector.offset[__type] = \ 30 offsetof(struct nf_flow_key, __field) 31 32 static void nf_flow_rule_lwt_match(struct nf_flow_match *match, 33 struct ip_tunnel_info *tun_info) 34 { 35 struct nf_flow_key *mask = &match->mask; 36 struct nf_flow_key *key = &match->key; 37 unsigned int enc_keys; 38 39 if (!tun_info || !(tun_info->mode & IP_TUNNEL_INFO_TX)) 40 return; 41 42 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_CONTROL, enc_control); 43 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id); 44 key->enc_key_id.keyid = tunnel_id_to_key32(tun_info->key.tun_id); 45 mask->enc_key_id.keyid = 0xffffffff; 46 enc_keys = BIT(FLOW_DISSECTOR_KEY_ENC_KEYID) | 47 BIT(FLOW_DISSECTOR_KEY_ENC_CONTROL); 48 49 if (ip_tunnel_info_af(tun_info) == AF_INET) { 50 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, 51 enc_ipv4); 52 key->enc_ipv4.src = tun_info->key.u.ipv4.dst; 53 key->enc_ipv4.dst = tun_info->key.u.ipv4.src; 54 if (key->enc_ipv4.src) 55 mask->enc_ipv4.src = 0xffffffff; 56 if (key->enc_ipv4.dst) 57 mask->enc_ipv4.dst = 0xffffffff; 58 enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS); 59 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 60 } else { 61 memcpy(&key->enc_ipv6.src, &tun_info->key.u.ipv6.dst, 62 sizeof(struct in6_addr)); 63 memcpy(&key->enc_ipv6.dst, &tun_info->key.u.ipv6.src, 64 sizeof(struct in6_addr)); 65 if (memcmp(&key->enc_ipv6.src, &in6addr_any, 66 sizeof(struct in6_addr))) 67 memset(&mask->enc_ipv6.src, 0xff, 68 sizeof(struct in6_addr)); 69 if (memcmp(&key->enc_ipv6.dst, &in6addr_any, 70 sizeof(struct in6_addr))) 71 memset(&mask->enc_ipv6.dst, 0xff, 72 sizeof(struct in6_addr)); 73 enc_keys |= BIT(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS); 74 key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 75 } 76 77 match->dissector.used_keys |= enc_keys; 78 } 79 80 static void nf_flow_rule_vlan_match(struct flow_dissector_key_vlan *key, 81 struct flow_dissector_key_vlan *mask, 82 u16 vlan_id, __be16 proto) 83 { 84 key->vlan_id = vlan_id; 85 mask->vlan_id = VLAN_VID_MASK; 86 key->vlan_tpid = proto; 87 mask->vlan_tpid = 0xffff; 88 } 89 90 static int nf_flow_rule_match(struct nf_flow_match *match, 91 const struct flow_offload_tuple *tuple, 92 struct dst_entry *other_dst) 93 { 94 struct nf_flow_key *mask = &match->mask; 95 struct nf_flow_key *key = &match->key; 96 struct ip_tunnel_info *tun_info; 97 bool vlan_encap = false; 98 99 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_META, meta); 100 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CONTROL, control); 101 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_BASIC, basic); 102 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4); 103 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6); 104 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_TCP, tcp); 105 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_PORTS, tp); 106 107 if (other_dst && other_dst->lwtstate) { 108 tun_info = lwt_tun_info(other_dst->lwtstate); 109 nf_flow_rule_lwt_match(match, tun_info); 110 } 111 112 if (tuple->xmit_type == FLOW_OFFLOAD_XMIT_TC) 113 key->meta.ingress_ifindex = tuple->tc.iifidx; 114 else 115 key->meta.ingress_ifindex = tuple->iifidx; 116 117 mask->meta.ingress_ifindex = 0xffffffff; 118 119 if (tuple->encap_num > 0 && !(tuple->in_vlan_ingress & BIT(0)) && 120 tuple->encap[0].proto == htons(ETH_P_8021Q)) { 121 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, vlan); 122 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 123 tuple->encap[0].id, 124 tuple->encap[0].proto); 125 vlan_encap = true; 126 } 127 128 if (tuple->encap_num > 1 && !(tuple->in_vlan_ingress & BIT(1)) && 129 tuple->encap[1].proto == htons(ETH_P_8021Q)) { 130 if (vlan_encap) { 131 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_CVLAN, 132 cvlan); 133 nf_flow_rule_vlan_match(&key->cvlan, &mask->cvlan, 134 tuple->encap[1].id, 135 tuple->encap[1].proto); 136 } else { 137 NF_FLOW_DISSECTOR(match, FLOW_DISSECTOR_KEY_VLAN, 138 vlan); 139 nf_flow_rule_vlan_match(&key->vlan, &mask->vlan, 140 tuple->encap[1].id, 141 tuple->encap[1].proto); 142 } 143 } 144 145 switch (tuple->l3proto) { 146 case AF_INET: 147 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; 148 key->basic.n_proto = htons(ETH_P_IP); 149 key->ipv4.src = tuple->src_v4.s_addr; 150 mask->ipv4.src = 0xffffffff; 151 key->ipv4.dst = tuple->dst_v4.s_addr; 152 mask->ipv4.dst = 0xffffffff; 153 break; 154 case AF_INET6: 155 key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS; 156 key->basic.n_proto = htons(ETH_P_IPV6); 157 key->ipv6.src = tuple->src_v6; 158 memset(&mask->ipv6.src, 0xff, sizeof(mask->ipv6.src)); 159 key->ipv6.dst = tuple->dst_v6; 160 memset(&mask->ipv6.dst, 0xff, sizeof(mask->ipv6.dst)); 161 break; 162 default: 163 return -EOPNOTSUPP; 164 } 165 mask->control.addr_type = 0xffff; 166 match->dissector.used_keys |= BIT(key->control.addr_type); 167 mask->basic.n_proto = 0xffff; 168 169 switch (tuple->l4proto) { 170 case IPPROTO_TCP: 171 key->tcp.flags = 0; 172 mask->tcp.flags = cpu_to_be16(be32_to_cpu(TCP_FLAG_RST | TCP_FLAG_FIN) >> 16); 173 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_TCP); 174 break; 175 case IPPROTO_UDP: 176 case IPPROTO_GRE: 177 break; 178 default: 179 return -EOPNOTSUPP; 180 } 181 182 key->basic.ip_proto = tuple->l4proto; 183 mask->basic.ip_proto = 0xff; 184 185 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_META) | 186 BIT(FLOW_DISSECTOR_KEY_CONTROL) | 187 BIT(FLOW_DISSECTOR_KEY_BASIC); 188 189 switch (tuple->l4proto) { 190 case IPPROTO_TCP: 191 case IPPROTO_UDP: 192 key->tp.src = tuple->src_port; 193 mask->tp.src = 0xffff; 194 key->tp.dst = tuple->dst_port; 195 mask->tp.dst = 0xffff; 196 197 match->dissector.used_keys |= BIT(FLOW_DISSECTOR_KEY_PORTS); 198 break; 199 } 200 201 return 0; 202 } 203 204 static void flow_offload_mangle(struct flow_action_entry *entry, 205 enum flow_action_mangle_base htype, u32 offset, 206 const __be32 *value, const __be32 *mask) 207 { 208 entry->id = FLOW_ACTION_MANGLE; 209 entry->mangle.htype = htype; 210 entry->mangle.offset = offset; 211 memcpy(&entry->mangle.mask, mask, sizeof(u32)); 212 memcpy(&entry->mangle.val, value, sizeof(u32)); 213 } 214 215 static inline struct flow_action_entry * 216 flow_action_entry_next(struct nf_flow_rule *flow_rule) 217 { 218 int i = flow_rule->rule->action.num_entries++; 219 220 return &flow_rule->rule->action.entries[i]; 221 } 222 223 static int flow_offload_eth_src(struct net *net, 224 const struct flow_offload *flow, 225 enum flow_offload_tuple_dir dir, 226 struct nf_flow_rule *flow_rule) 227 { 228 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 229 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 230 const struct flow_offload_tuple *other_tuple, *this_tuple; 231 struct net_device *dev = NULL; 232 const unsigned char *addr; 233 u32 mask, val; 234 u16 val16; 235 236 this_tuple = &flow->tuplehash[dir].tuple; 237 238 switch (this_tuple->xmit_type) { 239 case FLOW_OFFLOAD_XMIT_DIRECT: 240 addr = this_tuple->out.h_source; 241 break; 242 case FLOW_OFFLOAD_XMIT_NEIGH: 243 other_tuple = &flow->tuplehash[!dir].tuple; 244 dev = dev_get_by_index(net, other_tuple->iifidx); 245 if (!dev) 246 return -ENOENT; 247 248 addr = dev->dev_addr; 249 break; 250 default: 251 return -EOPNOTSUPP; 252 } 253 254 mask = ~0xffff0000; 255 memcpy(&val16, addr, 2); 256 val = val16 << 16; 257 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 258 &val, &mask); 259 260 mask = ~0xffffffff; 261 memcpy(&val, addr + 2, 4); 262 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 8, 263 &val, &mask); 264 265 dev_put(dev); 266 267 return 0; 268 } 269 270 static int flow_offload_eth_dst(struct net *net, 271 const struct flow_offload *flow, 272 enum flow_offload_tuple_dir dir, 273 struct nf_flow_rule *flow_rule) 274 { 275 struct flow_action_entry *entry0 = flow_action_entry_next(flow_rule); 276 struct flow_action_entry *entry1 = flow_action_entry_next(flow_rule); 277 const struct flow_offload_tuple *other_tuple, *this_tuple; 278 const struct dst_entry *dst_cache; 279 unsigned char ha[ETH_ALEN]; 280 struct neighbour *n; 281 const void *daddr; 282 u32 mask, val; 283 u8 nud_state; 284 u16 val16; 285 286 this_tuple = &flow->tuplehash[dir].tuple; 287 288 switch (this_tuple->xmit_type) { 289 case FLOW_OFFLOAD_XMIT_DIRECT: 290 ether_addr_copy(ha, this_tuple->out.h_dest); 291 break; 292 case FLOW_OFFLOAD_XMIT_NEIGH: 293 other_tuple = &flow->tuplehash[!dir].tuple; 294 daddr = &other_tuple->src_v4; 295 dst_cache = this_tuple->dst_cache; 296 n = dst_neigh_lookup(dst_cache, daddr); 297 if (!n) 298 return -ENOENT; 299 300 read_lock_bh(&n->lock); 301 nud_state = n->nud_state; 302 ether_addr_copy(ha, n->ha); 303 read_unlock_bh(&n->lock); 304 neigh_release(n); 305 306 if (!(nud_state & NUD_VALID)) 307 return -ENOENT; 308 break; 309 default: 310 return -EOPNOTSUPP; 311 } 312 313 mask = ~0xffffffff; 314 memcpy(&val, ha, 4); 315 flow_offload_mangle(entry0, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 0, 316 &val, &mask); 317 318 mask = ~0x0000ffff; 319 memcpy(&val16, ha + 4, 2); 320 val = val16; 321 flow_offload_mangle(entry1, FLOW_ACT_MANGLE_HDR_TYPE_ETH, 4, 322 &val, &mask); 323 324 return 0; 325 } 326 327 static void flow_offload_ipv4_snat(struct net *net, 328 const struct flow_offload *flow, 329 enum flow_offload_tuple_dir dir, 330 struct nf_flow_rule *flow_rule) 331 { 332 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 333 u32 mask = ~htonl(0xffffffff); 334 __be32 addr; 335 u32 offset; 336 337 switch (dir) { 338 case FLOW_OFFLOAD_DIR_ORIGINAL: 339 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v4.s_addr; 340 offset = offsetof(struct iphdr, saddr); 341 break; 342 case FLOW_OFFLOAD_DIR_REPLY: 343 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v4.s_addr; 344 offset = offsetof(struct iphdr, daddr); 345 break; 346 default: 347 return; 348 } 349 350 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 351 &addr, &mask); 352 } 353 354 static void flow_offload_ipv4_dnat(struct net *net, 355 const struct flow_offload *flow, 356 enum flow_offload_tuple_dir dir, 357 struct nf_flow_rule *flow_rule) 358 { 359 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 360 u32 mask = ~htonl(0xffffffff); 361 __be32 addr; 362 u32 offset; 363 364 switch (dir) { 365 case FLOW_OFFLOAD_DIR_ORIGINAL: 366 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v4.s_addr; 367 offset = offsetof(struct iphdr, daddr); 368 break; 369 case FLOW_OFFLOAD_DIR_REPLY: 370 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v4.s_addr; 371 offset = offsetof(struct iphdr, saddr); 372 break; 373 default: 374 return; 375 } 376 377 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP4, offset, 378 &addr, &mask); 379 } 380 381 static void flow_offload_ipv6_mangle(struct nf_flow_rule *flow_rule, 382 unsigned int offset, 383 const __be32 *addr, const __be32 *mask) 384 { 385 struct flow_action_entry *entry; 386 int i, j; 387 388 for (i = 0, j = 0; i < sizeof(struct in6_addr) / sizeof(u32); i += sizeof(u32), j++) { 389 entry = flow_action_entry_next(flow_rule); 390 flow_offload_mangle(entry, FLOW_ACT_MANGLE_HDR_TYPE_IP6, 391 offset + i, &addr[j], mask); 392 } 393 } 394 395 static void flow_offload_ipv6_snat(struct net *net, 396 const struct flow_offload *flow, 397 enum flow_offload_tuple_dir dir, 398 struct nf_flow_rule *flow_rule) 399 { 400 u32 mask = ~htonl(0xffffffff); 401 const __be32 *addr; 402 u32 offset; 403 404 switch (dir) { 405 case FLOW_OFFLOAD_DIR_ORIGINAL: 406 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_v6.s6_addr32; 407 offset = offsetof(struct ipv6hdr, saddr); 408 break; 409 case FLOW_OFFLOAD_DIR_REPLY: 410 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_v6.s6_addr32; 411 offset = offsetof(struct ipv6hdr, daddr); 412 break; 413 default: 414 return; 415 } 416 417 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 418 } 419 420 static void flow_offload_ipv6_dnat(struct net *net, 421 const struct flow_offload *flow, 422 enum flow_offload_tuple_dir dir, 423 struct nf_flow_rule *flow_rule) 424 { 425 u32 mask = ~htonl(0xffffffff); 426 const __be32 *addr; 427 u32 offset; 428 429 switch (dir) { 430 case FLOW_OFFLOAD_DIR_ORIGINAL: 431 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_v6.s6_addr32; 432 offset = offsetof(struct ipv6hdr, daddr); 433 break; 434 case FLOW_OFFLOAD_DIR_REPLY: 435 addr = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_v6.s6_addr32; 436 offset = offsetof(struct ipv6hdr, saddr); 437 break; 438 default: 439 return; 440 } 441 442 flow_offload_ipv6_mangle(flow_rule, offset, addr, &mask); 443 } 444 445 static int flow_offload_l4proto(const struct flow_offload *flow) 446 { 447 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 448 u8 type = 0; 449 450 switch (protonum) { 451 case IPPROTO_TCP: 452 type = FLOW_ACT_MANGLE_HDR_TYPE_TCP; 453 break; 454 case IPPROTO_UDP: 455 type = FLOW_ACT_MANGLE_HDR_TYPE_UDP; 456 break; 457 default: 458 break; 459 } 460 461 return type; 462 } 463 464 static void flow_offload_port_snat(struct net *net, 465 const struct flow_offload *flow, 466 enum flow_offload_tuple_dir dir, 467 struct nf_flow_rule *flow_rule) 468 { 469 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 470 u32 mask, port; 471 u32 offset; 472 473 switch (dir) { 474 case FLOW_OFFLOAD_DIR_ORIGINAL: 475 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.dst_port); 476 offset = 0; /* offsetof(struct tcphdr, source); */ 477 port = htonl(port << 16); 478 mask = ~htonl(0xffff0000); 479 break; 480 case FLOW_OFFLOAD_DIR_REPLY: 481 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.src_port); 482 offset = 0; /* offsetof(struct tcphdr, dest); */ 483 port = htonl(port); 484 mask = ~htonl(0xffff); 485 break; 486 default: 487 return; 488 } 489 490 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 491 &port, &mask); 492 } 493 494 static void flow_offload_port_dnat(struct net *net, 495 const struct flow_offload *flow, 496 enum flow_offload_tuple_dir dir, 497 struct nf_flow_rule *flow_rule) 498 { 499 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 500 u32 mask, port; 501 u32 offset; 502 503 switch (dir) { 504 case FLOW_OFFLOAD_DIR_ORIGINAL: 505 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_REPLY].tuple.src_port); 506 offset = 0; /* offsetof(struct tcphdr, dest); */ 507 port = htonl(port); 508 mask = ~htonl(0xffff); 509 break; 510 case FLOW_OFFLOAD_DIR_REPLY: 511 port = ntohs(flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.dst_port); 512 offset = 0; /* offsetof(struct tcphdr, source); */ 513 port = htonl(port << 16); 514 mask = ~htonl(0xffff0000); 515 break; 516 default: 517 return; 518 } 519 520 flow_offload_mangle(entry, flow_offload_l4proto(flow), offset, 521 &port, &mask); 522 } 523 524 static void flow_offload_ipv4_checksum(struct net *net, 525 const struct flow_offload *flow, 526 struct nf_flow_rule *flow_rule) 527 { 528 u8 protonum = flow->tuplehash[FLOW_OFFLOAD_DIR_ORIGINAL].tuple.l4proto; 529 struct flow_action_entry *entry = flow_action_entry_next(flow_rule); 530 531 entry->id = FLOW_ACTION_CSUM; 532 entry->csum_flags = TCA_CSUM_UPDATE_FLAG_IPV4HDR; 533 534 switch (protonum) { 535 case IPPROTO_TCP: 536 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_TCP; 537 break; 538 case IPPROTO_UDP: 539 entry->csum_flags |= TCA_CSUM_UPDATE_FLAG_UDP; 540 break; 541 } 542 } 543 544 static void flow_offload_redirect(struct net *net, 545 const struct flow_offload *flow, 546 enum flow_offload_tuple_dir dir, 547 struct nf_flow_rule *flow_rule) 548 { 549 const struct flow_offload_tuple *this_tuple, *other_tuple; 550 struct flow_action_entry *entry; 551 struct net_device *dev; 552 int ifindex; 553 554 this_tuple = &flow->tuplehash[dir].tuple; 555 switch (this_tuple->xmit_type) { 556 case FLOW_OFFLOAD_XMIT_DIRECT: 557 this_tuple = &flow->tuplehash[dir].tuple; 558 ifindex = this_tuple->out.hw_ifidx; 559 break; 560 case FLOW_OFFLOAD_XMIT_NEIGH: 561 other_tuple = &flow->tuplehash[!dir].tuple; 562 ifindex = other_tuple->iifidx; 563 break; 564 default: 565 return; 566 } 567 568 dev = dev_get_by_index(net, ifindex); 569 if (!dev) 570 return; 571 572 entry = flow_action_entry_next(flow_rule); 573 entry->id = FLOW_ACTION_REDIRECT; 574 entry->dev = dev; 575 } 576 577 static void flow_offload_encap_tunnel(const struct flow_offload *flow, 578 enum flow_offload_tuple_dir dir, 579 struct nf_flow_rule *flow_rule) 580 { 581 const struct flow_offload_tuple *this_tuple; 582 struct flow_action_entry *entry; 583 struct dst_entry *dst; 584 585 this_tuple = &flow->tuplehash[dir].tuple; 586 if (this_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 587 return; 588 589 dst = this_tuple->dst_cache; 590 if (dst && dst->lwtstate) { 591 struct ip_tunnel_info *tun_info; 592 593 tun_info = lwt_tun_info(dst->lwtstate); 594 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 595 entry = flow_action_entry_next(flow_rule); 596 entry->id = FLOW_ACTION_TUNNEL_ENCAP; 597 entry->tunnel = tun_info; 598 } 599 } 600 } 601 602 static void flow_offload_decap_tunnel(const struct flow_offload *flow, 603 enum flow_offload_tuple_dir dir, 604 struct nf_flow_rule *flow_rule) 605 { 606 const struct flow_offload_tuple *other_tuple; 607 struct flow_action_entry *entry; 608 struct dst_entry *dst; 609 610 other_tuple = &flow->tuplehash[!dir].tuple; 611 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_DIRECT) 612 return; 613 614 dst = other_tuple->dst_cache; 615 if (dst && dst->lwtstate) { 616 struct ip_tunnel_info *tun_info; 617 618 tun_info = lwt_tun_info(dst->lwtstate); 619 if (tun_info && (tun_info->mode & IP_TUNNEL_INFO_TX)) { 620 entry = flow_action_entry_next(flow_rule); 621 entry->id = FLOW_ACTION_TUNNEL_DECAP; 622 } 623 } 624 } 625 626 static int 627 nf_flow_rule_route_common(struct net *net, const struct flow_offload *flow, 628 enum flow_offload_tuple_dir dir, 629 struct nf_flow_rule *flow_rule) 630 { 631 const struct flow_offload_tuple *other_tuple; 632 const struct flow_offload_tuple *tuple; 633 int i; 634 635 flow_offload_decap_tunnel(flow, dir, flow_rule); 636 flow_offload_encap_tunnel(flow, dir, flow_rule); 637 638 if (flow_offload_eth_src(net, flow, dir, flow_rule) < 0 || 639 flow_offload_eth_dst(net, flow, dir, flow_rule) < 0) 640 return -1; 641 642 tuple = &flow->tuplehash[dir].tuple; 643 644 for (i = 0; i < tuple->encap_num; i++) { 645 struct flow_action_entry *entry; 646 647 if (tuple->in_vlan_ingress & BIT(i)) 648 continue; 649 650 if (tuple->encap[i].proto == htons(ETH_P_8021Q)) { 651 entry = flow_action_entry_next(flow_rule); 652 entry->id = FLOW_ACTION_VLAN_POP; 653 } 654 } 655 656 other_tuple = &flow->tuplehash[!dir].tuple; 657 658 for (i = 0; i < other_tuple->encap_num; i++) { 659 struct flow_action_entry *entry; 660 661 if (other_tuple->in_vlan_ingress & BIT(i)) 662 continue; 663 664 entry = flow_action_entry_next(flow_rule); 665 666 switch (other_tuple->encap[i].proto) { 667 case htons(ETH_P_PPP_SES): 668 entry->id = FLOW_ACTION_PPPOE_PUSH; 669 entry->pppoe.sid = other_tuple->encap[i].id; 670 break; 671 case htons(ETH_P_8021Q): 672 entry->id = FLOW_ACTION_VLAN_PUSH; 673 entry->vlan.vid = other_tuple->encap[i].id; 674 entry->vlan.proto = other_tuple->encap[i].proto; 675 break; 676 } 677 } 678 679 return 0; 680 } 681 682 int nf_flow_rule_route_ipv4(struct net *net, const struct flow_offload *flow, 683 enum flow_offload_tuple_dir dir, 684 struct nf_flow_rule *flow_rule) 685 { 686 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 687 return -1; 688 689 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 690 flow_offload_ipv4_snat(net, flow, dir, flow_rule); 691 flow_offload_port_snat(net, flow, dir, flow_rule); 692 } 693 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 694 flow_offload_ipv4_dnat(net, flow, dir, flow_rule); 695 flow_offload_port_dnat(net, flow, dir, flow_rule); 696 } 697 if (test_bit(NF_FLOW_SNAT, &flow->flags) || 698 test_bit(NF_FLOW_DNAT, &flow->flags)) 699 flow_offload_ipv4_checksum(net, flow, flow_rule); 700 701 flow_offload_redirect(net, flow, dir, flow_rule); 702 703 return 0; 704 } 705 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv4); 706 707 int nf_flow_rule_route_ipv6(struct net *net, const struct flow_offload *flow, 708 enum flow_offload_tuple_dir dir, 709 struct nf_flow_rule *flow_rule) 710 { 711 if (nf_flow_rule_route_common(net, flow, dir, flow_rule) < 0) 712 return -1; 713 714 if (test_bit(NF_FLOW_SNAT, &flow->flags)) { 715 flow_offload_ipv6_snat(net, flow, dir, flow_rule); 716 flow_offload_port_snat(net, flow, dir, flow_rule); 717 } 718 if (test_bit(NF_FLOW_DNAT, &flow->flags)) { 719 flow_offload_ipv6_dnat(net, flow, dir, flow_rule); 720 flow_offload_port_dnat(net, flow, dir, flow_rule); 721 } 722 723 flow_offload_redirect(net, flow, dir, flow_rule); 724 725 return 0; 726 } 727 EXPORT_SYMBOL_GPL(nf_flow_rule_route_ipv6); 728 729 #define NF_FLOW_RULE_ACTION_MAX 16 730 731 static struct nf_flow_rule * 732 nf_flow_offload_rule_alloc(struct net *net, 733 const struct flow_offload_work *offload, 734 enum flow_offload_tuple_dir dir) 735 { 736 const struct nf_flowtable *flowtable = offload->flowtable; 737 const struct flow_offload_tuple *tuple, *other_tuple; 738 const struct flow_offload *flow = offload->flow; 739 struct dst_entry *other_dst = NULL; 740 struct nf_flow_rule *flow_rule; 741 int err = -ENOMEM; 742 743 flow_rule = kzalloc(sizeof(*flow_rule), GFP_KERNEL); 744 if (!flow_rule) 745 goto err_flow; 746 747 flow_rule->rule = flow_rule_alloc(NF_FLOW_RULE_ACTION_MAX); 748 if (!flow_rule->rule) 749 goto err_flow_rule; 750 751 flow_rule->rule->match.dissector = &flow_rule->match.dissector; 752 flow_rule->rule->match.mask = &flow_rule->match.mask; 753 flow_rule->rule->match.key = &flow_rule->match.key; 754 755 tuple = &flow->tuplehash[dir].tuple; 756 other_tuple = &flow->tuplehash[!dir].tuple; 757 if (other_tuple->xmit_type == FLOW_OFFLOAD_XMIT_NEIGH) 758 other_dst = other_tuple->dst_cache; 759 760 err = nf_flow_rule_match(&flow_rule->match, tuple, other_dst); 761 if (err < 0) 762 goto err_flow_match; 763 764 flow_rule->rule->action.num_entries = 0; 765 if (flowtable->type->action(net, flow, dir, flow_rule) < 0) 766 goto err_flow_match; 767 768 return flow_rule; 769 770 err_flow_match: 771 kfree(flow_rule->rule); 772 err_flow_rule: 773 kfree(flow_rule); 774 err_flow: 775 return NULL; 776 } 777 778 static void __nf_flow_offload_destroy(struct nf_flow_rule *flow_rule) 779 { 780 struct flow_action_entry *entry; 781 int i; 782 783 for (i = 0; i < flow_rule->rule->action.num_entries; i++) { 784 entry = &flow_rule->rule->action.entries[i]; 785 if (entry->id != FLOW_ACTION_REDIRECT) 786 continue; 787 788 dev_put(entry->dev); 789 } 790 kfree(flow_rule->rule); 791 kfree(flow_rule); 792 } 793 794 static void nf_flow_offload_destroy(struct nf_flow_rule *flow_rule[]) 795 { 796 int i; 797 798 for (i = 0; i < FLOW_OFFLOAD_DIR_MAX; i++) 799 __nf_flow_offload_destroy(flow_rule[i]); 800 } 801 802 static int nf_flow_offload_alloc(const struct flow_offload_work *offload, 803 struct nf_flow_rule *flow_rule[]) 804 { 805 struct net *net = read_pnet(&offload->flowtable->net); 806 807 flow_rule[0] = nf_flow_offload_rule_alloc(net, offload, 808 FLOW_OFFLOAD_DIR_ORIGINAL); 809 if (!flow_rule[0]) 810 return -ENOMEM; 811 812 flow_rule[1] = nf_flow_offload_rule_alloc(net, offload, 813 FLOW_OFFLOAD_DIR_REPLY); 814 if (!flow_rule[1]) { 815 __nf_flow_offload_destroy(flow_rule[0]); 816 return -ENOMEM; 817 } 818 819 return 0; 820 } 821 822 static void nf_flow_offload_init(struct flow_cls_offload *cls_flow, 823 __be16 proto, int priority, 824 enum flow_cls_command cmd, 825 const struct flow_offload_tuple *tuple, 826 struct netlink_ext_ack *extack) 827 { 828 cls_flow->common.protocol = proto; 829 cls_flow->common.prio = priority; 830 cls_flow->common.extack = extack; 831 cls_flow->command = cmd; 832 cls_flow->cookie = (unsigned long)tuple; 833 } 834 835 static int nf_flow_offload_tuple(struct nf_flowtable *flowtable, 836 struct flow_offload *flow, 837 struct nf_flow_rule *flow_rule, 838 enum flow_offload_tuple_dir dir, 839 int priority, int cmd, 840 struct flow_stats *stats, 841 struct list_head *block_cb_list) 842 { 843 struct flow_cls_offload cls_flow = {}; 844 struct flow_block_cb *block_cb; 845 struct netlink_ext_ack extack; 846 __be16 proto = ETH_P_ALL; 847 int err, i = 0; 848 849 nf_flow_offload_init(&cls_flow, proto, priority, cmd, 850 &flow->tuplehash[dir].tuple, &extack); 851 if (cmd == FLOW_CLS_REPLACE) 852 cls_flow.rule = flow_rule->rule; 853 854 down_read(&flowtable->flow_block_lock); 855 list_for_each_entry(block_cb, block_cb_list, list) { 856 err = block_cb->cb(TC_SETUP_CLSFLOWER, &cls_flow, 857 block_cb->cb_priv); 858 if (err < 0) 859 continue; 860 861 i++; 862 } 863 up_read(&flowtable->flow_block_lock); 864 865 if (cmd == FLOW_CLS_STATS) 866 memcpy(stats, &cls_flow.stats, sizeof(*stats)); 867 868 return i; 869 } 870 871 static int flow_offload_tuple_add(struct flow_offload_work *offload, 872 struct nf_flow_rule *flow_rule, 873 enum flow_offload_tuple_dir dir) 874 { 875 return nf_flow_offload_tuple(offload->flowtable, offload->flow, 876 flow_rule, dir, 877 offload->flowtable->priority, 878 FLOW_CLS_REPLACE, NULL, 879 &offload->flowtable->flow_block.cb_list); 880 } 881 882 static void flow_offload_tuple_del(struct flow_offload_work *offload, 883 enum flow_offload_tuple_dir dir) 884 { 885 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 886 offload->flowtable->priority, 887 FLOW_CLS_DESTROY, NULL, 888 &offload->flowtable->flow_block.cb_list); 889 } 890 891 static int flow_offload_rule_add(struct flow_offload_work *offload, 892 struct nf_flow_rule *flow_rule[]) 893 { 894 int ok_count = 0; 895 896 ok_count += flow_offload_tuple_add(offload, flow_rule[0], 897 FLOW_OFFLOAD_DIR_ORIGINAL); 898 ok_count += flow_offload_tuple_add(offload, flow_rule[1], 899 FLOW_OFFLOAD_DIR_REPLY); 900 if (ok_count == 0) 901 return -ENOENT; 902 903 return 0; 904 } 905 906 static void flow_offload_work_add(struct flow_offload_work *offload) 907 { 908 struct nf_flow_rule *flow_rule[FLOW_OFFLOAD_DIR_MAX]; 909 int err; 910 911 err = nf_flow_offload_alloc(offload, flow_rule); 912 if (err < 0) 913 return; 914 915 err = flow_offload_rule_add(offload, flow_rule); 916 if (err < 0) 917 goto out; 918 919 set_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 920 921 out: 922 nf_flow_offload_destroy(flow_rule); 923 } 924 925 static void flow_offload_work_del(struct flow_offload_work *offload) 926 { 927 clear_bit(IPS_HW_OFFLOAD_BIT, &offload->flow->ct->status); 928 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_ORIGINAL); 929 flow_offload_tuple_del(offload, FLOW_OFFLOAD_DIR_REPLY); 930 set_bit(NF_FLOW_HW_DEAD, &offload->flow->flags); 931 } 932 933 static void flow_offload_tuple_stats(struct flow_offload_work *offload, 934 enum flow_offload_tuple_dir dir, 935 struct flow_stats *stats) 936 { 937 nf_flow_offload_tuple(offload->flowtable, offload->flow, NULL, dir, 938 offload->flowtable->priority, 939 FLOW_CLS_STATS, stats, 940 &offload->flowtable->flow_block.cb_list); 941 } 942 943 static void flow_offload_work_stats(struct flow_offload_work *offload) 944 { 945 struct flow_stats stats[FLOW_OFFLOAD_DIR_MAX] = {}; 946 u64 lastused; 947 948 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_ORIGINAL, &stats[0]); 949 flow_offload_tuple_stats(offload, FLOW_OFFLOAD_DIR_REPLY, &stats[1]); 950 951 lastused = max_t(u64, stats[0].lastused, stats[1].lastused); 952 offload->flow->timeout = max_t(u64, offload->flow->timeout, 953 lastused + flow_offload_get_timeout(offload->flow)); 954 955 if (offload->flowtable->flags & NF_FLOWTABLE_COUNTER) { 956 if (stats[0].pkts) 957 nf_ct_acct_add(offload->flow->ct, 958 FLOW_OFFLOAD_DIR_ORIGINAL, 959 stats[0].pkts, stats[0].bytes); 960 if (stats[1].pkts) 961 nf_ct_acct_add(offload->flow->ct, 962 FLOW_OFFLOAD_DIR_REPLY, 963 stats[1].pkts, stats[1].bytes); 964 } 965 } 966 967 static void flow_offload_work_handler(struct work_struct *work) 968 { 969 struct flow_offload_work *offload; 970 struct net *net; 971 972 offload = container_of(work, struct flow_offload_work, work); 973 net = read_pnet(&offload->flowtable->net); 974 switch (offload->cmd) { 975 case FLOW_CLS_REPLACE: 976 flow_offload_work_add(offload); 977 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_add); 978 break; 979 case FLOW_CLS_DESTROY: 980 flow_offload_work_del(offload); 981 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_del); 982 break; 983 case FLOW_CLS_STATS: 984 flow_offload_work_stats(offload); 985 NF_FLOW_TABLE_STAT_DEC_ATOMIC(net, count_wq_stats); 986 break; 987 default: 988 WARN_ON_ONCE(1); 989 } 990 991 clear_bit(NF_FLOW_HW_PENDING, &offload->flow->flags); 992 kfree(offload); 993 } 994 995 static void flow_offload_queue_work(struct flow_offload_work *offload) 996 { 997 struct net *net = read_pnet(&offload->flowtable->net); 998 999 if (offload->cmd == FLOW_CLS_REPLACE) { 1000 NF_FLOW_TABLE_STAT_INC(net, count_wq_add); 1001 queue_work(nf_flow_offload_add_wq, &offload->work); 1002 } else if (offload->cmd == FLOW_CLS_DESTROY) { 1003 NF_FLOW_TABLE_STAT_INC(net, count_wq_del); 1004 queue_work(nf_flow_offload_del_wq, &offload->work); 1005 } else { 1006 NF_FLOW_TABLE_STAT_INC(net, count_wq_stats); 1007 queue_work(nf_flow_offload_stats_wq, &offload->work); 1008 } 1009 } 1010 1011 static struct flow_offload_work * 1012 nf_flow_offload_work_alloc(struct nf_flowtable *flowtable, 1013 struct flow_offload *flow, unsigned int cmd) 1014 { 1015 struct flow_offload_work *offload; 1016 1017 if (test_and_set_bit(NF_FLOW_HW_PENDING, &flow->flags)) 1018 return NULL; 1019 1020 offload = kmalloc(sizeof(struct flow_offload_work), GFP_ATOMIC); 1021 if (!offload) { 1022 clear_bit(NF_FLOW_HW_PENDING, &flow->flags); 1023 return NULL; 1024 } 1025 1026 offload->cmd = cmd; 1027 offload->flow = flow; 1028 offload->flowtable = flowtable; 1029 INIT_WORK(&offload->work, flow_offload_work_handler); 1030 1031 return offload; 1032 } 1033 1034 1035 void nf_flow_offload_add(struct nf_flowtable *flowtable, 1036 struct flow_offload *flow) 1037 { 1038 struct flow_offload_work *offload; 1039 1040 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_REPLACE); 1041 if (!offload) 1042 return; 1043 1044 flow_offload_queue_work(offload); 1045 } 1046 1047 void nf_flow_offload_del(struct nf_flowtable *flowtable, 1048 struct flow_offload *flow) 1049 { 1050 struct flow_offload_work *offload; 1051 1052 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_DESTROY); 1053 if (!offload) 1054 return; 1055 1056 set_bit(NF_FLOW_HW_DYING, &flow->flags); 1057 flow_offload_queue_work(offload); 1058 } 1059 1060 void nf_flow_offload_stats(struct nf_flowtable *flowtable, 1061 struct flow_offload *flow) 1062 { 1063 struct flow_offload_work *offload; 1064 __s32 delta; 1065 1066 delta = nf_flow_timeout_delta(flow->timeout); 1067 if ((delta >= (9 * flow_offload_get_timeout(flow)) / 10)) 1068 return; 1069 1070 offload = nf_flow_offload_work_alloc(flowtable, flow, FLOW_CLS_STATS); 1071 if (!offload) 1072 return; 1073 1074 flow_offload_queue_work(offload); 1075 } 1076 1077 void nf_flow_table_offload_flush_cleanup(struct nf_flowtable *flowtable) 1078 { 1079 if (nf_flowtable_hw_offload(flowtable)) { 1080 flush_workqueue(nf_flow_offload_del_wq); 1081 nf_flow_table_gc_run(flowtable); 1082 } 1083 } 1084 1085 void nf_flow_table_offload_flush(struct nf_flowtable *flowtable) 1086 { 1087 if (nf_flowtable_hw_offload(flowtable)) { 1088 flush_workqueue(nf_flow_offload_add_wq); 1089 flush_workqueue(nf_flow_offload_del_wq); 1090 flush_workqueue(nf_flow_offload_stats_wq); 1091 } 1092 } 1093 1094 static int nf_flow_table_block_setup(struct nf_flowtable *flowtable, 1095 struct flow_block_offload *bo, 1096 enum flow_block_command cmd) 1097 { 1098 struct flow_block_cb *block_cb, *next; 1099 int err = 0; 1100 1101 switch (cmd) { 1102 case FLOW_BLOCK_BIND: 1103 list_splice(&bo->cb_list, &flowtable->flow_block.cb_list); 1104 break; 1105 case FLOW_BLOCK_UNBIND: 1106 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1107 list_del(&block_cb->list); 1108 flow_block_cb_free(block_cb); 1109 } 1110 break; 1111 default: 1112 WARN_ON_ONCE(1); 1113 err = -EOPNOTSUPP; 1114 } 1115 1116 return err; 1117 } 1118 1119 static void nf_flow_table_block_offload_init(struct flow_block_offload *bo, 1120 struct net *net, 1121 enum flow_block_command cmd, 1122 struct nf_flowtable *flowtable, 1123 struct netlink_ext_ack *extack) 1124 { 1125 memset(bo, 0, sizeof(*bo)); 1126 bo->net = net; 1127 bo->block = &flowtable->flow_block; 1128 bo->command = cmd; 1129 bo->binder_type = FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS; 1130 bo->extack = extack; 1131 bo->cb_list_head = &flowtable->flow_block.cb_list; 1132 INIT_LIST_HEAD(&bo->cb_list); 1133 } 1134 1135 static void nf_flow_table_indr_cleanup(struct flow_block_cb *block_cb) 1136 { 1137 struct nf_flowtable *flowtable = block_cb->indr.data; 1138 struct net_device *dev = block_cb->indr.dev; 1139 1140 nf_flow_table_gc_cleanup(flowtable, dev); 1141 down_write(&flowtable->flow_block_lock); 1142 list_del(&block_cb->list); 1143 list_del(&block_cb->driver_list); 1144 flow_block_cb_free(block_cb); 1145 up_write(&flowtable->flow_block_lock); 1146 } 1147 1148 static int nf_flow_table_indr_offload_cmd(struct flow_block_offload *bo, 1149 struct nf_flowtable *flowtable, 1150 struct net_device *dev, 1151 enum flow_block_command cmd, 1152 struct netlink_ext_ack *extack) 1153 { 1154 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1155 extack); 1156 1157 return flow_indr_dev_setup_offload(dev, NULL, TC_SETUP_FT, flowtable, bo, 1158 nf_flow_table_indr_cleanup); 1159 } 1160 1161 static int nf_flow_table_offload_cmd(struct flow_block_offload *bo, 1162 struct nf_flowtable *flowtable, 1163 struct net_device *dev, 1164 enum flow_block_command cmd, 1165 struct netlink_ext_ack *extack) 1166 { 1167 int err; 1168 1169 nf_flow_table_block_offload_init(bo, dev_net(dev), cmd, flowtable, 1170 extack); 1171 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_FT, bo); 1172 if (err < 0) 1173 return err; 1174 1175 return 0; 1176 } 1177 1178 int nf_flow_table_offload_setup(struct nf_flowtable *flowtable, 1179 struct net_device *dev, 1180 enum flow_block_command cmd) 1181 { 1182 struct netlink_ext_ack extack = {}; 1183 struct flow_block_offload bo; 1184 int err; 1185 1186 if (!nf_flowtable_hw_offload(flowtable)) 1187 return 0; 1188 1189 if (dev->netdev_ops->ndo_setup_tc) 1190 err = nf_flow_table_offload_cmd(&bo, flowtable, dev, cmd, 1191 &extack); 1192 else 1193 err = nf_flow_table_indr_offload_cmd(&bo, flowtable, dev, cmd, 1194 &extack); 1195 if (err < 0) 1196 return err; 1197 1198 return nf_flow_table_block_setup(flowtable, &bo, cmd); 1199 } 1200 EXPORT_SYMBOL_GPL(nf_flow_table_offload_setup); 1201 1202 int nf_flow_table_offload_init(void) 1203 { 1204 nf_flow_offload_add_wq = alloc_workqueue("nf_ft_offload_add", 1205 WQ_UNBOUND | WQ_SYSFS, 0); 1206 if (!nf_flow_offload_add_wq) 1207 return -ENOMEM; 1208 1209 nf_flow_offload_del_wq = alloc_workqueue("nf_ft_offload_del", 1210 WQ_UNBOUND | WQ_SYSFS, 0); 1211 if (!nf_flow_offload_del_wq) 1212 goto err_del_wq; 1213 1214 nf_flow_offload_stats_wq = alloc_workqueue("nf_ft_offload_stats", 1215 WQ_UNBOUND | WQ_SYSFS, 0); 1216 if (!nf_flow_offload_stats_wq) 1217 goto err_stats_wq; 1218 1219 return 0; 1220 1221 err_stats_wq: 1222 destroy_workqueue(nf_flow_offload_del_wq); 1223 err_del_wq: 1224 destroy_workqueue(nf_flow_offload_add_wq); 1225 return -ENOMEM; 1226 } 1227 1228 void nf_flow_table_offload_exit(void) 1229 { 1230 destroy_workqueue(nf_flow_offload_add_wq); 1231 destroy_workqueue(nf_flow_offload_del_wq); 1232 destroy_workqueue(nf_flow_offload_stats_wq); 1233 } 1234