230 	match->value._field = fm.key->_tcfield;				\
231 	match->mask._field = fm.mask->_tcfield;				\
240 				     struct efx_tc_match *match,  in efx_tc_flower_parse_match()  argument
243 	struct flow_dissector *dissector = rule->match.dissector;  in efx_tc_flower_parse_match()
269 			match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;  in efx_tc_flower_parse_match()
270 			match->mask.ip_frag = true;  in efx_tc_flower_parse_match()
273 			match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;  in efx_tc_flower_parse_match()
274 			match->mask.ip_firstfrag = true;  in efx_tc_flower_parse_match()
307 	if (!IS_ALL_ONES(match->mask.eth_proto) ||  in efx_tc_flower_parse_match()
308 	    !(match->value.eth_proto == htons(ETH_P_IP) ||  in efx_tc_flower_parse_match()
309 	      match->value.eth_proto == htons(ETH_P_IPV6)))  in efx_tc_flower_parse_match()
327 			match->value.vlan_proto[0] = fm.key->vlan_tpid;  in efx_tc_flower_parse_match()
328 			match->mask.vlan_proto[0] = fm.mask->vlan_tpid;  in efx_tc_flower_parse_match()
329 			match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |  in efx_tc_flower_parse_match()
331 			match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |  in efx_tc_flower_parse_match()
341 			match->value.vlan_proto[1] = fm.key->vlan_tpid;  in efx_tc_flower_parse_match()
342 			match->mask.vlan_proto[1] = fm.mask->vlan_tpid;  in efx_tc_flower_parse_match()
343 			match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |  in efx_tc_flower_parse_match()
345 			match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |  in efx_tc_flower_parse_match()
354 		ether_addr_copy(match->value.eth_saddr, fm.key->src);  in efx_tc_flower_parse_match()
355 		ether_addr_copy(match->value.eth_daddr, fm.key->dst);  in efx_tc_flower_parse_match()
356 		ether_addr_copy(match->mask.eth_saddr, fm.mask->src);  in efx_tc_flower_parse_match()
357 		ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);  in efx_tc_flower_parse_match()
362 	if ((match->value.ip_proto != IPPROTO_UDP &&  in efx_tc_flower_parse_match()
363 	     match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))  in efx_tc_flower_parse_match()
443 		match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);  in efx_tc_flower_parse_match()
444 		match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);  in efx_tc_flower_parse_match()
445 		match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);  in efx_tc_flower_parse_match()
446 		match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);  in efx_tc_flower_parse_match()
454 		match->value.ct_mark = fm.key->ct_mark;  in efx_tc_flower_parse_match()
455 		match->mask.ct_mark = fm.mask->ct_mark;  in efx_tc_flower_parse_match()
456 		match->value.ct_zone = fm.key->ct_zone;  in efx_tc_flower_parse_match()
457 		match->mask.ct_zone = fm.mask->ct_zone;  in efx_tc_flower_parse_match()
494 					    struct efx_tc_match *match,  in efx_tc_flower_record_encap_match()  argument
511 	if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {  in efx_tc_flower_record_encap_match()
512 		if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {  in efx_tc_flower_record_encap_match()
517 		if (!IS_ALL_ONES(match->mask.enc_src_ip)) {  in efx_tc_flower_record_encap_match()
523 		if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||  in efx_tc_flower_record_encap_match()
524 		    !ipv6_addr_any(&match->mask.enc_src_ip6)) {  in efx_tc_flower_record_encap_match()
531 		if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {  in efx_tc_flower_record_encap_match()
536 		if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {  in efx_tc_flower_record_encap_match()
543 	if (!IS_ALL_ONES(match->mask.enc_dport)) {  in efx_tc_flower_record_encap_match()
547 	if (match->mask.enc_sport || match->mask.enc_ip_tos) {  in efx_tc_flower_record_encap_match()
548 		struct efx_tc_match pmatch = *match;  in efx_tc_flower_record_encap_match()
560 						      match->mask.enc_ip_tos,  in efx_tc_flower_record_encap_match()
561 						      match->mask.enc_sport,  in efx_tc_flower_record_encap_match()
567 	if (match->mask.enc_ip_ttl) {  in efx_tc_flower_record_encap_match()
573 	rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,  in efx_tc_flower_record_encap_match()
574 					    match->mask.enc_sport, extack);  in efx_tc_flower_record_encap_match()
583 	encap->src_ip = match->value.enc_src_ip;  in efx_tc_flower_record_encap_match()
584 	encap->dst_ip = match->value.enc_dst_ip;  in efx_tc_flower_record_encap_match()
586 	encap->src_ip6 = match->value.enc_src_ip6;  in efx_tc_flower_record_encap_match()
587 	encap->dst_ip6 = match->value.enc_dst_ip6;  in efx_tc_flower_record_encap_match()
589 	encap->udp_dport = match->value.enc_dport;  in efx_tc_flower_record_encap_match()
591 	encap->ip_tos = match->value.enc_ip_tos;  in efx_tc_flower_record_encap_match()
592 	encap->ip_tos_mask = match->mask.enc_ip_tos;  in efx_tc_flower_record_encap_match()
594 	encap->udp_sport = match->value.enc_sport;  in efx_tc_flower_record_encap_match()
595 	encap->udp_sport_mask = match->mask.enc_sport;  in efx_tc_flower_record_encap_match()
673 	match->encap = encap;  in efx_tc_flower_record_encap_match()
744 	if (rule->match.rid)  in efx_tc_delete_rule()
745 		efx_tc_put_recirc_id(efx, rule->match.rid);  in efx_tc_delete_rule()
746 	if (rule->match.encap)  in efx_tc_delete_rule()
747 		efx_tc_flower_release_encap_match(efx, rule->match.encap);  in efx_tc_delete_rule()
849 				    struct efx_tc_match *match)  in efx_tc_rule_is_lhs_rule()  argument
865 			if (!match->mask.ct_state_trk || !match->value.ct_state_trk)  in efx_tc_rule_is_lhs_rule()
1148 			 struct efx_tc_match *match)  in efx_tc_mangle()  argument
1228 			if (match->mask.ip_ttl != U8_MAX) {  in efx_tc_mangle()
1231 						       match->mask.ip_ttl);  in efx_tc_mangle()
1238 			if (match->value.ip_ttl == 0) {  in efx_tc_mangle()
1253 			tr_ttl = match->value.ip_ttl - 1;  in efx_tc_mangle()
1287 			if (match->mask.ip_ttl != U8_MAX) {  in efx_tc_mangle()
1290 						       match->mask.ip_ttl);  in efx_tc_mangle()
1297 			if (match->value.ip_ttl == 0) {  in efx_tc_mangle()
1312 			tr_ttl = match->value.ip_ttl - 1;  in efx_tc_mangle()
1367 	struct efx_tc_match match;  in efx_tc_flower_replace_foreign()  local
1373 	memset(&match, 0, sizeof(match));  in efx_tc_flower_replace_foreign()
1374 	rc = efx_tc_flower_parse_match(efx, fr, &match, NULL);  in efx_tc_flower_replace_foreign()
1387 	match.value.ingress_port = rc;  in efx_tc_flower_replace_foreign()
1388 	match.mask.ingress_port = ~0;  in efx_tc_flower_replace_foreign()
1400 		match.rid = rid;  in efx_tc_flower_replace_foreign()
1401 		match.value.recirc_id = rid->fw_id;  in efx_tc_flower_replace_foreign()
1403 	match.mask.recirc_id = 0xff;  in efx_tc_flower_replace_foreign()
1408 	if (match.mask.ct_state_trk && match.value.ct_state_trk &&  in efx_tc_flower_replace_foreign()
1409 	    match.mask.ct_state_est && match.value.ct_state_est)  in efx_tc_flower_replace_foreign()
1410 		match.mask.ct_state_trk = 0;  in efx_tc_flower_replace_foreign()
1416 	if (match.mask.ct_state_est && !match.value.ct_state_est) {  in efx_tc_flower_replace_foreign()
1417 		if (match.value.tcp_syn_fin_rst) {  in efx_tc_flower_replace_foreign()
1422 		match.mask.tcp_syn_fin_rst = true;  in efx_tc_flower_replace_foreign()
1445 	rc = efx_mae_match_check_caps(efx, &match.mask, NULL);  in efx_tc_flower_replace_foreign()
1449 	if (efx_tc_match_is_encap(&match.mask)) {  in efx_tc_flower_replace_foreign()
1468 		rc = efx_tc_flower_record_encap_match(efx, &match, type,  in efx_tc_flower_replace_foreign()
1632 	rule->match = match;  in efx_tc_flower_replace_foreign()
1643 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace_foreign()
1657 	if (match.rid)  in efx_tc_flower_replace_foreign()
1658 		efx_tc_put_recirc_id(efx, match.rid);  in efx_tc_flower_replace_foreign()
1669 	if (match.encap)  in efx_tc_flower_replace_foreign()
1670 		efx_tc_flower_release_encap_match(efx, match.encap);  in efx_tc_flower_replace_foreign()
1677 				     struct efx_tc_match *match,  in efx_tc_flower_replace_lhs()  argument
1690 	if (match->mask.ct_state_trk && match->value.ct_state_trk) {  in efx_tc_flower_replace_lhs()
1695 	match->mask.ct_state_trk = 0;  in efx_tc_flower_replace_lhs()
1696 	match->value.ct_state_trk = 0;  in efx_tc_flower_replace_lhs()
1698 	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);  in efx_tc_flower_replace_lhs()
1728 	rule->match = *match;  in efx_tc_flower_replace_lhs()
1762 	struct efx_tc_match match;  in efx_tc_flower_replace()  local
1789 	memset(&match, 0, sizeof(match));  in efx_tc_flower_replace()
1795 	match.value.ingress_port = rc;  in efx_tc_flower_replace()
1796 	match.mask.ingress_port = ~0;  in efx_tc_flower_replace()
1797 	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);  in efx_tc_flower_replace()
1800 	if (efx_tc_match_is_encap(&match.mask)) {  in efx_tc_flower_replace()
1805 	if (efx_tc_rule_is_lhs_rule(fr, &match))  in efx_tc_flower_replace()
1806 		return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,  in efx_tc_flower_replace()
1834 		match.rid = rid;  in efx_tc_flower_replace()
1835 		match.value.recirc_id = rid->fw_id;  in efx_tc_flower_replace()
1837 	match.mask.recirc_id = 0xff;  in efx_tc_flower_replace()
1842 	if (match.mask.ct_state_trk && match.value.ct_state_trk &&  in efx_tc_flower_replace()
1843 	    match.mask.ct_state_est && match.value.ct_state_est)  in efx_tc_flower_replace()
1844 		match.mask.ct_state_trk = 0;  in efx_tc_flower_replace()
1850 	if (match.mask.ct_state_est && !match.value.ct_state_est) {  in efx_tc_flower_replace()
1851 		if (match.value.tcp_syn_fin_rst) {  in efx_tc_flower_replace()
1856 		match.mask.tcp_syn_fin_rst = true;  in efx_tc_flower_replace()
1859 	rc = efx_mae_match_check_caps(efx, &match.mask, extack);  in efx_tc_flower_replace()
2104 			rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);  in efx_tc_flower_replace()
2178 	rule->match = match;  in efx_tc_flower_replace()
2198 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace()
2212 	if (match.rid)  in efx_tc_flower_replace()
2213 		efx_tc_put_recirc_id(efx, match.rid);  in efx_tc_flower_replace()
2244 		if (lhs_rule->match.encap)  in efx_tc_flower_destroy()
2245 			efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);  in efx_tc_flower_destroy()
2344 	struct efx_tc_match *match = &rule->match;  in efx_tc_configure_default_rule()  local
2348 	match->value.ingress_port = ing_port;  in efx_tc_configure_default_rule()
2349 	match->mask.ingress_port = ~0;  in efx_tc_configure_default_rule()
2363 	rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,  in efx_tc_configure_default_rule()