128bf2672SBrett Creeley // SPDX-License-Identifier: GPL-2.0 228bf2672SBrett Creeley /* Copyright (C) 2018-2020, Intel Corporation. */ 328bf2672SBrett Creeley 428bf2672SBrett Creeley #include "ice.h" 528bf2672SBrett Creeley 628bf2672SBrett Creeley /** 728bf2672SBrett Creeley * ice_is_arfs_active - helper to check is aRFS is active 828bf2672SBrett Creeley * @vsi: VSI to check 928bf2672SBrett Creeley */ 1028bf2672SBrett Creeley static bool ice_is_arfs_active(struct ice_vsi *vsi) 1128bf2672SBrett Creeley { 1228bf2672SBrett Creeley return !!vsi->arfs_fltr_list; 1328bf2672SBrett Creeley } 1428bf2672SBrett Creeley 1528bf2672SBrett Creeley /** 1628bf2672SBrett Creeley * ice_is_arfs_using_perfect_flow - check if aRFS has active perfect filters 1728bf2672SBrett Creeley * @hw: pointer to the HW structure 1828bf2672SBrett Creeley * @flow_type: flow type as Flow Director understands it 1928bf2672SBrett Creeley * 2028bf2672SBrett Creeley * Flow Director will query this function to see if aRFS is currently using 2128bf2672SBrett Creeley * the specified flow_type for perfect (4-tuple) filters. 2228bf2672SBrett Creeley */ 2328bf2672SBrett Creeley bool 2428bf2672SBrett Creeley ice_is_arfs_using_perfect_flow(struct ice_hw *hw, enum ice_fltr_ptype flow_type) 2528bf2672SBrett Creeley { 2628bf2672SBrett Creeley struct ice_arfs_active_fltr_cntrs *arfs_fltr_cntrs; 2728bf2672SBrett Creeley struct ice_pf *pf = hw->back; 2828bf2672SBrett Creeley struct ice_vsi *vsi; 2928bf2672SBrett Creeley 3028bf2672SBrett Creeley vsi = ice_get_main_vsi(pf); 3128bf2672SBrett Creeley if (!vsi) 3228bf2672SBrett Creeley return false; 3328bf2672SBrett Creeley 3428bf2672SBrett Creeley arfs_fltr_cntrs = vsi->arfs_fltr_cntrs; 3528bf2672SBrett Creeley 3628bf2672SBrett Creeley /* active counters can be updated by multiple CPUs */ 3728bf2672SBrett Creeley smp_mb__before_atomic(); 3828bf2672SBrett Creeley switch (flow_type) { 3928bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_UDP: 4028bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_udpv4_cnt) > 0; 4128bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_UDP: 4228bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_udpv6_cnt) > 0; 4328bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_TCP: 4428bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_tcpv4_cnt) > 0; 4528bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_TCP: 4628bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_tcpv6_cnt) > 0; 4728bf2672SBrett Creeley default: 4828bf2672SBrett Creeley return false; 4928bf2672SBrett Creeley } 5028bf2672SBrett Creeley } 5128bf2672SBrett Creeley 5228bf2672SBrett Creeley /** 5328bf2672SBrett Creeley * ice_arfs_update_active_fltr_cntrs - update active filter counters for aRFS 5428bf2672SBrett Creeley * @vsi: VSI that aRFS is active on 5528bf2672SBrett Creeley * @entry: aRFS entry used to change counters 5628bf2672SBrett Creeley * @add: true to increment counter, false to decrement 5728bf2672SBrett Creeley */ 5828bf2672SBrett Creeley static void 5928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(struct ice_vsi *vsi, 6028bf2672SBrett Creeley struct ice_arfs_entry *entry, bool add) 6128bf2672SBrett Creeley { 6228bf2672SBrett Creeley struct ice_arfs_active_fltr_cntrs *fltr_cntrs = vsi->arfs_fltr_cntrs; 6328bf2672SBrett Creeley 6428bf2672SBrett Creeley switch (entry->fltr_info.flow_type) { 6528bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_TCP: 6628bf2672SBrett Creeley if (add) 6728bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_tcpv4_cnt); 6828bf2672SBrett Creeley else 6928bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_tcpv4_cnt); 7028bf2672SBrett Creeley break; 7128bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_TCP: 7228bf2672SBrett Creeley if (add) 7328bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_tcpv6_cnt); 7428bf2672SBrett Creeley else 7528bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_tcpv6_cnt); 7628bf2672SBrett Creeley break; 7728bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_UDP: 7828bf2672SBrett Creeley if (add) 7928bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_udpv4_cnt); 8028bf2672SBrett Creeley else 8128bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_udpv4_cnt); 8228bf2672SBrett Creeley break; 8328bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_UDP: 8428bf2672SBrett Creeley if (add) 8528bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_udpv6_cnt); 8628bf2672SBrett Creeley else 8728bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_udpv6_cnt); 8828bf2672SBrett Creeley break; 8928bf2672SBrett Creeley default: 9028bf2672SBrett Creeley dev_err(ice_pf_to_dev(vsi->back), "aRFS: Failed to update filter counters, invalid filter type %d\n", 9128bf2672SBrett Creeley entry->fltr_info.flow_type); 9228bf2672SBrett Creeley } 9328bf2672SBrett Creeley } 9428bf2672SBrett Creeley 9528bf2672SBrett Creeley /** 9628bf2672SBrett Creeley * ice_arfs_del_flow_rules - delete the rules passed in from HW 9728bf2672SBrett Creeley * @vsi: VSI for the flow rules that need to be deleted 9828bf2672SBrett Creeley * @del_list_head: head of the list of ice_arfs_entry(s) for rule deletion 9928bf2672SBrett Creeley * 10028bf2672SBrett Creeley * Loop through the delete list passed in and remove the rules from HW. After 10128bf2672SBrett Creeley * each rule is deleted, disconnect and free the ice_arfs_entry because it is no 10228bf2672SBrett Creeley * longer being referenced by the aRFS hash table. 10328bf2672SBrett Creeley */ 10428bf2672SBrett Creeley static void 10528bf2672SBrett Creeley ice_arfs_del_flow_rules(struct ice_vsi *vsi, struct hlist_head *del_list_head) 10628bf2672SBrett Creeley { 10728bf2672SBrett Creeley struct ice_arfs_entry *e; 10828bf2672SBrett Creeley struct hlist_node *n; 10928bf2672SBrett Creeley struct device *dev; 11028bf2672SBrett Creeley 11128bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back); 11228bf2672SBrett Creeley 11328bf2672SBrett Creeley hlist_for_each_entry_safe(e, n, del_list_head, list_entry) { 11428bf2672SBrett Creeley int result; 11528bf2672SBrett Creeley 11628bf2672SBrett Creeley result = ice_fdir_write_fltr(vsi->back, &e->fltr_info, false, 11728bf2672SBrett Creeley false); 11828bf2672SBrett Creeley if (!result) 11928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, e, false); 12028bf2672SBrett Creeley else 12128bf2672SBrett Creeley dev_dbg(dev, "Unable to delete aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n", 12228bf2672SBrett Creeley result, e->fltr_state, e->fltr_info.fltr_id, 12328bf2672SBrett Creeley e->flow_id, e->fltr_info.q_index); 12428bf2672SBrett Creeley 12528bf2672SBrett Creeley /* The aRFS hash table is no longer referencing this entry */ 12628bf2672SBrett Creeley hlist_del(&e->list_entry); 12728bf2672SBrett Creeley devm_kfree(dev, e); 12828bf2672SBrett Creeley } 12928bf2672SBrett Creeley } 13028bf2672SBrett Creeley 13128bf2672SBrett Creeley /** 13228bf2672SBrett Creeley * ice_arfs_add_flow_rules - add the rules passed in from HW 13328bf2672SBrett Creeley * @vsi: VSI for the flow rules that need to be added 13428bf2672SBrett Creeley * @add_list_head: head of the list of ice_arfs_entry_ptr(s) for rule addition 13528bf2672SBrett Creeley * 13628bf2672SBrett Creeley * Loop through the add list passed in and remove the rules from HW. After each 13728bf2672SBrett Creeley * rule is added, disconnect and free the ice_arfs_entry_ptr node. Don't free 13828bf2672SBrett Creeley * the ice_arfs_entry(s) because they are still being referenced in the aRFS 13928bf2672SBrett Creeley * hash table. 14028bf2672SBrett Creeley */ 14128bf2672SBrett Creeley static void 14228bf2672SBrett Creeley ice_arfs_add_flow_rules(struct ice_vsi *vsi, struct hlist_head *add_list_head) 14328bf2672SBrett Creeley { 14428bf2672SBrett Creeley struct ice_arfs_entry_ptr *ep; 14528bf2672SBrett Creeley struct hlist_node *n; 14628bf2672SBrett Creeley struct device *dev; 14728bf2672SBrett Creeley 14828bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back); 14928bf2672SBrett Creeley 15028bf2672SBrett Creeley hlist_for_each_entry_safe(ep, n, add_list_head, list_entry) { 15128bf2672SBrett Creeley int result; 15228bf2672SBrett Creeley 15328bf2672SBrett Creeley result = ice_fdir_write_fltr(vsi->back, 15428bf2672SBrett Creeley &ep->arfs_entry->fltr_info, true, 15528bf2672SBrett Creeley false); 15628bf2672SBrett Creeley if (!result) 15728bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, ep->arfs_entry, 15828bf2672SBrett Creeley true); 15928bf2672SBrett Creeley else 16028bf2672SBrett Creeley dev_dbg(dev, "Unable to add aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n", 16128bf2672SBrett Creeley result, ep->arfs_entry->fltr_state, 16228bf2672SBrett Creeley ep->arfs_entry->fltr_info.fltr_id, 16328bf2672SBrett Creeley ep->arfs_entry->flow_id, 16428bf2672SBrett Creeley ep->arfs_entry->fltr_info.q_index); 16528bf2672SBrett Creeley 16628bf2672SBrett Creeley hlist_del(&ep->list_entry); 16728bf2672SBrett Creeley devm_kfree(dev, ep); 16828bf2672SBrett Creeley } 16928bf2672SBrett Creeley } 17028bf2672SBrett Creeley 17128bf2672SBrett Creeley /** 17228bf2672SBrett Creeley * ice_arfs_is_flow_expired - check if the aRFS entry has expired 17328bf2672SBrett Creeley * @vsi: VSI containing the aRFS entry 17428bf2672SBrett Creeley * @arfs_entry: aRFS entry that's being checked for expiration 17528bf2672SBrett Creeley * 17628bf2672SBrett Creeley * Return true if the flow has expired, else false. This function should be used 17728bf2672SBrett Creeley * to determine whether or not an aRFS entry should be removed from the hardware 17828bf2672SBrett Creeley * and software structures. 17928bf2672SBrett Creeley */ 18028bf2672SBrett Creeley static bool 18128bf2672SBrett Creeley ice_arfs_is_flow_expired(struct ice_vsi *vsi, struct ice_arfs_entry *arfs_entry) 18228bf2672SBrett Creeley { 18328bf2672SBrett Creeley #define ICE_ARFS_TIME_DELTA_EXPIRATION msecs_to_jiffies(5000) 18428bf2672SBrett Creeley if (rps_may_expire_flow(vsi->netdev, arfs_entry->fltr_info.q_index, 18528bf2672SBrett Creeley arfs_entry->flow_id, 18628bf2672SBrett Creeley arfs_entry->fltr_info.fltr_id)) 18728bf2672SBrett Creeley return true; 18828bf2672SBrett Creeley 18928bf2672SBrett Creeley /* expiration timer only used for UDP filters */ 19028bf2672SBrett Creeley if (arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV4_UDP && 19128bf2672SBrett Creeley arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV6_UDP) 19228bf2672SBrett Creeley return false; 19328bf2672SBrett Creeley 19428bf2672SBrett Creeley return time_in_range64(arfs_entry->time_activated + 19528bf2672SBrett Creeley ICE_ARFS_TIME_DELTA_EXPIRATION, 19628bf2672SBrett Creeley arfs_entry->time_activated, get_jiffies_64()); 19728bf2672SBrett Creeley } 19828bf2672SBrett Creeley 19928bf2672SBrett Creeley /** 20028bf2672SBrett Creeley * ice_arfs_update_flow_rules - add/delete aRFS rules in HW 20128bf2672SBrett Creeley * @vsi: the VSI to be forwarded to 20228bf2672SBrett Creeley * @idx: index into the table of aRFS filter lists. Obtained from skb->hash 20328bf2672SBrett Creeley * @add_list: list to populate with filters to be added to Flow Director 20428bf2672SBrett Creeley * @del_list: list to populate with filters to be deleted from Flow Director 20528bf2672SBrett Creeley * 20628bf2672SBrett Creeley * Iterate over the hlist at the index given in the aRFS hash table and 20728bf2672SBrett Creeley * determine if there are any aRFS entries that need to be either added or 20828bf2672SBrett Creeley * deleted in the HW. If the aRFS entry is marked as ICE_ARFS_INACTIVE the 20928bf2672SBrett Creeley * filter needs to be added to HW, else if it's marked as ICE_ARFS_ACTIVE and 21028bf2672SBrett Creeley * the flow has expired delete the filter from HW. The caller of this function 21128bf2672SBrett Creeley * is expected to add/delete rules on the add_list/del_list respectively. 21228bf2672SBrett Creeley */ 21328bf2672SBrett Creeley static void 21428bf2672SBrett Creeley ice_arfs_update_flow_rules(struct ice_vsi *vsi, u16 idx, 21528bf2672SBrett Creeley struct hlist_head *add_list, 21628bf2672SBrett Creeley struct hlist_head *del_list) 21728bf2672SBrett Creeley { 21828bf2672SBrett Creeley struct ice_arfs_entry *e; 21928bf2672SBrett Creeley struct hlist_node *n; 22028bf2672SBrett Creeley struct device *dev; 22128bf2672SBrett Creeley 22228bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back); 22328bf2672SBrett Creeley 22428bf2672SBrett Creeley /* go through the aRFS hlist at this idx and check for needed updates */ 22528bf2672SBrett Creeley hlist_for_each_entry_safe(e, n, &vsi->arfs_fltr_list[idx], list_entry) 22628bf2672SBrett Creeley /* check if filter needs to be added to HW */ 22728bf2672SBrett Creeley if (e->fltr_state == ICE_ARFS_INACTIVE) { 22828bf2672SBrett Creeley enum ice_fltr_ptype flow_type = e->fltr_info.flow_type; 22928bf2672SBrett Creeley struct ice_arfs_entry_ptr *ep = 23028bf2672SBrett Creeley devm_kzalloc(dev, sizeof(*ep), GFP_ATOMIC); 23128bf2672SBrett Creeley 23228bf2672SBrett Creeley if (!ep) 23328bf2672SBrett Creeley continue; 23428bf2672SBrett Creeley INIT_HLIST_NODE(&ep->list_entry); 23528bf2672SBrett Creeley /* reference aRFS entry to add HW filter */ 23628bf2672SBrett Creeley ep->arfs_entry = e; 23728bf2672SBrett Creeley hlist_add_head(&ep->list_entry, add_list); 23828bf2672SBrett Creeley e->fltr_state = ICE_ARFS_ACTIVE; 23928bf2672SBrett Creeley /* expiration timer only used for UDP flows */ 24028bf2672SBrett Creeley if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP || 24128bf2672SBrett Creeley flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP) 24228bf2672SBrett Creeley e->time_activated = get_jiffies_64(); 24328bf2672SBrett Creeley } else if (e->fltr_state == ICE_ARFS_ACTIVE) { 24428bf2672SBrett Creeley /* check if filter needs to be removed from HW */ 24528bf2672SBrett Creeley if (ice_arfs_is_flow_expired(vsi, e)) { 24628bf2672SBrett Creeley /* remove aRFS entry from hash table for delete 24728bf2672SBrett Creeley * and to prevent referencing it the next time 24828bf2672SBrett Creeley * through this hlist index 24928bf2672SBrett Creeley */ 25028bf2672SBrett Creeley hlist_del(&e->list_entry); 25128bf2672SBrett Creeley e->fltr_state = ICE_ARFS_TODEL; 25228bf2672SBrett Creeley /* save reference to aRFS entry for delete */ 25328bf2672SBrett Creeley hlist_add_head(&e->list_entry, del_list); 25428bf2672SBrett Creeley } 25528bf2672SBrett Creeley } 25628bf2672SBrett Creeley } 25728bf2672SBrett Creeley 25828bf2672SBrett Creeley /** 25928bf2672SBrett Creeley * ice_sync_arfs_fltrs - update all aRFS filters 26028bf2672SBrett Creeley * @pf: board private structure 26128bf2672SBrett Creeley */ 26228bf2672SBrett Creeley void ice_sync_arfs_fltrs(struct ice_pf *pf) 26328bf2672SBrett Creeley { 26428bf2672SBrett Creeley HLIST_HEAD(tmp_del_list); 26528bf2672SBrett Creeley HLIST_HEAD(tmp_add_list); 26628bf2672SBrett Creeley struct ice_vsi *pf_vsi; 26728bf2672SBrett Creeley unsigned int i; 26828bf2672SBrett Creeley 26928bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf); 27028bf2672SBrett Creeley if (!pf_vsi) 27128bf2672SBrett Creeley return; 27228bf2672SBrett Creeley 27328bf2672SBrett Creeley if (!ice_is_arfs_active(pf_vsi)) 27428bf2672SBrett Creeley return; 27528bf2672SBrett Creeley 27628bf2672SBrett Creeley spin_lock_bh(&pf_vsi->arfs_lock); 27728bf2672SBrett Creeley /* Once we process aRFS for the PF VSI get out */ 27828bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++) 27928bf2672SBrett Creeley ice_arfs_update_flow_rules(pf_vsi, i, &tmp_add_list, 28028bf2672SBrett Creeley &tmp_del_list); 28128bf2672SBrett Creeley spin_unlock_bh(&pf_vsi->arfs_lock); 28228bf2672SBrett Creeley 28328bf2672SBrett Creeley /* use list of ice_arfs_entry(s) for delete */ 28428bf2672SBrett Creeley ice_arfs_del_flow_rules(pf_vsi, &tmp_del_list); 28528bf2672SBrett Creeley 28628bf2672SBrett Creeley /* use list of ice_arfs_entry_ptr(s) for add */ 28728bf2672SBrett Creeley ice_arfs_add_flow_rules(pf_vsi, &tmp_add_list); 28828bf2672SBrett Creeley } 28928bf2672SBrett Creeley 29028bf2672SBrett Creeley /** 29128bf2672SBrett Creeley * ice_arfs_build_entry - builds an aRFS entry based on input 29228bf2672SBrett Creeley * @vsi: destination VSI for this flow 29328bf2672SBrett Creeley * @fk: flow dissector keys for creating the tuple 29428bf2672SBrett Creeley * @rxq_idx: Rx queue to steer this flow to 29528bf2672SBrett Creeley * @flow_id: passed down from the stack and saved for flow expiration 29628bf2672SBrett Creeley * 29728bf2672SBrett Creeley * returns an aRFS entry on success and NULL on failure 29828bf2672SBrett Creeley */ 29928bf2672SBrett Creeley static struct ice_arfs_entry * 30028bf2672SBrett Creeley ice_arfs_build_entry(struct ice_vsi *vsi, const struct flow_keys *fk, 30128bf2672SBrett Creeley u16 rxq_idx, u32 flow_id) 30228bf2672SBrett Creeley { 30328bf2672SBrett Creeley struct ice_arfs_entry *arfs_entry; 30428bf2672SBrett Creeley struct ice_fdir_fltr *fltr_info; 30528bf2672SBrett Creeley u8 ip_proto; 30628bf2672SBrett Creeley 30728bf2672SBrett Creeley arfs_entry = devm_kzalloc(ice_pf_to_dev(vsi->back), 30828bf2672SBrett Creeley sizeof(*arfs_entry), 30928bf2672SBrett Creeley GFP_ATOMIC | __GFP_NOWARN); 31028bf2672SBrett Creeley if (!arfs_entry) 31128bf2672SBrett Creeley return NULL; 31228bf2672SBrett Creeley 31328bf2672SBrett Creeley fltr_info = &arfs_entry->fltr_info; 31428bf2672SBrett Creeley fltr_info->q_index = rxq_idx; 31528bf2672SBrett Creeley fltr_info->dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX; 31628bf2672SBrett Creeley fltr_info->dest_vsi = vsi->idx; 31728bf2672SBrett Creeley ip_proto = fk->basic.ip_proto; 31828bf2672SBrett Creeley 31928bf2672SBrett Creeley if (fk->basic.n_proto == htons(ETH_P_IP)) { 32028bf2672SBrett Creeley fltr_info->ip.v4.proto = ip_proto; 32128bf2672SBrett Creeley fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ? 32228bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV4_TCP : 32328bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV4_UDP; 32428bf2672SBrett Creeley fltr_info->ip.v4.src_ip = fk->addrs.v4addrs.src; 32528bf2672SBrett Creeley fltr_info->ip.v4.dst_ip = fk->addrs.v4addrs.dst; 32628bf2672SBrett Creeley fltr_info->ip.v4.src_port = fk->ports.src; 32728bf2672SBrett Creeley fltr_info->ip.v4.dst_port = fk->ports.dst; 32828bf2672SBrett Creeley } else { /* ETH_P_IPV6 */ 32928bf2672SBrett Creeley fltr_info->ip.v6.proto = ip_proto; 33028bf2672SBrett Creeley fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ? 33128bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV6_TCP : 33228bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV6_UDP; 33328bf2672SBrett Creeley memcpy(&fltr_info->ip.v6.src_ip, &fk->addrs.v6addrs.src, 33428bf2672SBrett Creeley sizeof(struct in6_addr)); 33528bf2672SBrett Creeley memcpy(&fltr_info->ip.v6.dst_ip, &fk->addrs.v6addrs.dst, 33628bf2672SBrett Creeley sizeof(struct in6_addr)); 33728bf2672SBrett Creeley fltr_info->ip.v6.src_port = fk->ports.src; 33828bf2672SBrett Creeley fltr_info->ip.v6.dst_port = fk->ports.dst; 33928bf2672SBrett Creeley } 34028bf2672SBrett Creeley 34128bf2672SBrett Creeley arfs_entry->flow_id = flow_id; 34228bf2672SBrett Creeley fltr_info->fltr_id = 34328bf2672SBrett Creeley atomic_inc_return(vsi->arfs_last_fltr_id) % RPS_NO_FILTER; 34428bf2672SBrett Creeley 34528bf2672SBrett Creeley return arfs_entry; 34628bf2672SBrett Creeley } 34728bf2672SBrett Creeley 34828bf2672SBrett Creeley /** 34928bf2672SBrett Creeley * ice_arfs_is_perfect_flow_set - Check to see if perfect flow is set 35028bf2672SBrett Creeley * @hw: pointer to HW structure 35128bf2672SBrett Creeley * @l3_proto: ETH_P_IP or ETH_P_IPV6 in network order 35228bf2672SBrett Creeley * @l4_proto: IPPROTO_UDP or IPPROTO_TCP 35328bf2672SBrett Creeley * 35428bf2672SBrett Creeley * We only support perfect (4-tuple) filters for aRFS. This function allows aRFS 35528bf2672SBrett Creeley * to check if perfect (4-tuple) flow rules are currently in place by Flow 35628bf2672SBrett Creeley * Director. 35728bf2672SBrett Creeley */ 35828bf2672SBrett Creeley static bool 35928bf2672SBrett Creeley ice_arfs_is_perfect_flow_set(struct ice_hw *hw, __be16 l3_proto, u8 l4_proto) 36028bf2672SBrett Creeley { 36128bf2672SBrett Creeley unsigned long *perfect_fltr = hw->fdir_perfect_fltr; 36228bf2672SBrett Creeley 36328bf2672SBrett Creeley /* advanced Flow Director disabled, perfect filters always supported */ 36428bf2672SBrett Creeley if (!perfect_fltr) 36528bf2672SBrett Creeley return true; 36628bf2672SBrett Creeley 36728bf2672SBrett Creeley if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_UDP) 36828bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_UDP, perfect_fltr); 36928bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_TCP) 37028bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_TCP, perfect_fltr); 37128bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_UDP) 37228bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_UDP, perfect_fltr); 37328bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_TCP) 37428bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_TCP, perfect_fltr); 37528bf2672SBrett Creeley 37628bf2672SBrett Creeley return false; 37728bf2672SBrett Creeley } 37828bf2672SBrett Creeley 37928bf2672SBrett Creeley /** 38028bf2672SBrett Creeley * ice_rx_flow_steer - steer the Rx flow to where application is being run 38128bf2672SBrett Creeley * @netdev: ptr to the netdev being adjusted 38228bf2672SBrett Creeley * @skb: buffer with required header information 38328bf2672SBrett Creeley * @rxq_idx: queue to which the flow needs to move 38428bf2672SBrett Creeley * @flow_id: flow identifier provided by the netdev 38528bf2672SBrett Creeley * 38628bf2672SBrett Creeley * Based on the skb, rxq_idx, and flow_id passed in add/update an entry in the 38728bf2672SBrett Creeley * aRFS hash table. Iterate over one of the hlists in the aRFS hash table and 38828bf2672SBrett Creeley * if the flow_id already exists in the hash table but the rxq_idx has changed 38928bf2672SBrett Creeley * mark the entry as ICE_ARFS_INACTIVE so it can get updated in HW, else 39028bf2672SBrett Creeley * if the entry is marked as ICE_ARFS_TODEL delete it from the aRFS hash table. 39128bf2672SBrett Creeley * If neither of the previous conditions are true then add a new entry in the 39228bf2672SBrett Creeley * aRFS hash table, which gets set to ICE_ARFS_INACTIVE by default so it can be 39328bf2672SBrett Creeley * added to HW. 39428bf2672SBrett Creeley */ 39528bf2672SBrett Creeley int 39628bf2672SBrett Creeley ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb, 39728bf2672SBrett Creeley u16 rxq_idx, u32 flow_id) 39828bf2672SBrett Creeley { 39928bf2672SBrett Creeley struct ice_netdev_priv *np = netdev_priv(netdev); 40028bf2672SBrett Creeley struct ice_arfs_entry *arfs_entry; 40128bf2672SBrett Creeley struct ice_vsi *vsi = np->vsi; 40228bf2672SBrett Creeley struct flow_keys fk; 40328bf2672SBrett Creeley struct ice_pf *pf; 40428bf2672SBrett Creeley __be16 n_proto; 40528bf2672SBrett Creeley u8 ip_proto; 40628bf2672SBrett Creeley u16 idx; 40728bf2672SBrett Creeley int ret; 40828bf2672SBrett Creeley 40928bf2672SBrett Creeley /* failed to allocate memory for aRFS so don't crash */ 41028bf2672SBrett Creeley if (unlikely(!vsi->arfs_fltr_list)) 41128bf2672SBrett Creeley return -ENODEV; 41228bf2672SBrett Creeley 41328bf2672SBrett Creeley pf = vsi->back; 41428bf2672SBrett Creeley 41528bf2672SBrett Creeley if (skb->encapsulation) 41628bf2672SBrett Creeley return -EPROTONOSUPPORT; 41728bf2672SBrett Creeley 41828bf2672SBrett Creeley if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) 41928bf2672SBrett Creeley return -EPROTONOSUPPORT; 42028bf2672SBrett Creeley 42128bf2672SBrett Creeley n_proto = fk.basic.n_proto; 42228bf2672SBrett Creeley /* Support only IPV4 and IPV6 */ 42328bf2672SBrett Creeley if ((n_proto == htons(ETH_P_IP) && !ip_is_fragment(ip_hdr(skb))) || 42428bf2672SBrett Creeley n_proto == htons(ETH_P_IPV6)) 42528bf2672SBrett Creeley ip_proto = fk.basic.ip_proto; 42628bf2672SBrett Creeley else 42728bf2672SBrett Creeley return -EPROTONOSUPPORT; 42828bf2672SBrett Creeley 42928bf2672SBrett Creeley /* Support only TCP and UDP */ 43028bf2672SBrett Creeley if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP) 43128bf2672SBrett Creeley return -EPROTONOSUPPORT; 43228bf2672SBrett Creeley 43328bf2672SBrett Creeley /* only support 4-tuple filters for aRFS */ 43428bf2672SBrett Creeley if (!ice_arfs_is_perfect_flow_set(&pf->hw, n_proto, ip_proto)) 43528bf2672SBrett Creeley return -EOPNOTSUPP; 43628bf2672SBrett Creeley 43728bf2672SBrett Creeley /* choose the aRFS list bucket based on skb hash */ 43828bf2672SBrett Creeley idx = skb_get_hash_raw(skb) & ICE_ARFS_LST_MASK; 43928bf2672SBrett Creeley /* search for entry in the bucket */ 44028bf2672SBrett Creeley spin_lock_bh(&vsi->arfs_lock); 44128bf2672SBrett Creeley hlist_for_each_entry(arfs_entry, &vsi->arfs_fltr_list[idx], 44228bf2672SBrett Creeley list_entry) { 44328bf2672SBrett Creeley struct ice_fdir_fltr *fltr_info; 44428bf2672SBrett Creeley 44528bf2672SBrett Creeley /* keep searching for the already existing arfs_entry flow */ 44628bf2672SBrett Creeley if (arfs_entry->flow_id != flow_id) 44728bf2672SBrett Creeley continue; 44828bf2672SBrett Creeley 44928bf2672SBrett Creeley fltr_info = &arfs_entry->fltr_info; 45028bf2672SBrett Creeley ret = fltr_info->fltr_id; 45128bf2672SBrett Creeley 45228bf2672SBrett Creeley if (fltr_info->q_index == rxq_idx || 45328bf2672SBrett Creeley arfs_entry->fltr_state != ICE_ARFS_ACTIVE) 45428bf2672SBrett Creeley goto out; 45528bf2672SBrett Creeley 45628bf2672SBrett Creeley /* update the queue to forward to on an already existing flow */ 45728bf2672SBrett Creeley fltr_info->q_index = rxq_idx; 45828bf2672SBrett Creeley arfs_entry->fltr_state = ICE_ARFS_INACTIVE; 45928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, arfs_entry, false); 46028bf2672SBrett Creeley goto out_schedule_service_task; 46128bf2672SBrett Creeley } 46228bf2672SBrett Creeley 46328bf2672SBrett Creeley arfs_entry = ice_arfs_build_entry(vsi, &fk, rxq_idx, flow_id); 46428bf2672SBrett Creeley if (!arfs_entry) { 46528bf2672SBrett Creeley ret = -ENOMEM; 46628bf2672SBrett Creeley goto out; 46728bf2672SBrett Creeley } 46828bf2672SBrett Creeley 46928bf2672SBrett Creeley ret = arfs_entry->fltr_info.fltr_id; 47028bf2672SBrett Creeley INIT_HLIST_NODE(&arfs_entry->list_entry); 47128bf2672SBrett Creeley hlist_add_head(&arfs_entry->list_entry, &vsi->arfs_fltr_list[idx]); 47228bf2672SBrett Creeley out_schedule_service_task: 47328bf2672SBrett Creeley ice_service_task_schedule(pf); 47428bf2672SBrett Creeley out: 47528bf2672SBrett Creeley spin_unlock_bh(&vsi->arfs_lock); 47628bf2672SBrett Creeley return ret; 47728bf2672SBrett Creeley } 47828bf2672SBrett Creeley 47928bf2672SBrett Creeley /** 48028bf2672SBrett Creeley * ice_init_arfs_cntrs - initialize aRFS counter values 48128bf2672SBrett Creeley * @vsi: VSI that aRFS counters need to be initialized on 48228bf2672SBrett Creeley */ 48328bf2672SBrett Creeley static int ice_init_arfs_cntrs(struct ice_vsi *vsi) 48428bf2672SBrett Creeley { 48528bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF) 48628bf2672SBrett Creeley return -EINVAL; 48728bf2672SBrett Creeley 48828bf2672SBrett Creeley vsi->arfs_fltr_cntrs = kzalloc(sizeof(*vsi->arfs_fltr_cntrs), 48928bf2672SBrett Creeley GFP_KERNEL); 49028bf2672SBrett Creeley if (!vsi->arfs_fltr_cntrs) 49128bf2672SBrett Creeley return -ENOMEM; 49228bf2672SBrett Creeley 49328bf2672SBrett Creeley vsi->arfs_last_fltr_id = kzalloc(sizeof(*vsi->arfs_last_fltr_id), 49428bf2672SBrett Creeley GFP_KERNEL); 49528bf2672SBrett Creeley if (!vsi->arfs_last_fltr_id) { 49628bf2672SBrett Creeley kfree(vsi->arfs_fltr_cntrs); 49728bf2672SBrett Creeley vsi->arfs_fltr_cntrs = NULL; 49828bf2672SBrett Creeley return -ENOMEM; 49928bf2672SBrett Creeley } 50028bf2672SBrett Creeley 50128bf2672SBrett Creeley return 0; 50228bf2672SBrett Creeley } 50328bf2672SBrett Creeley 50428bf2672SBrett Creeley /** 50528bf2672SBrett Creeley * ice_init_arfs - initialize aRFS resources 50628bf2672SBrett Creeley * @vsi: the VSI to be forwarded to 50728bf2672SBrett Creeley */ 50828bf2672SBrett Creeley void ice_init_arfs(struct ice_vsi *vsi) 50928bf2672SBrett Creeley { 51028bf2672SBrett Creeley struct hlist_head *arfs_fltr_list; 51128bf2672SBrett Creeley unsigned int i; 51228bf2672SBrett Creeley 51328bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF) 51428bf2672SBrett Creeley return; 51528bf2672SBrett Creeley 51628bf2672SBrett Creeley arfs_fltr_list = kzalloc(sizeof(*arfs_fltr_list) * ICE_MAX_ARFS_LIST, 51728bf2672SBrett Creeley GFP_KERNEL); 51828bf2672SBrett Creeley if (!arfs_fltr_list) 51928bf2672SBrett Creeley return; 52028bf2672SBrett Creeley 52128bf2672SBrett Creeley if (ice_init_arfs_cntrs(vsi)) 52228bf2672SBrett Creeley goto free_arfs_fltr_list; 52328bf2672SBrett Creeley 52428bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++) 52528bf2672SBrett Creeley INIT_HLIST_HEAD(&arfs_fltr_list[i]); 52628bf2672SBrett Creeley 52728bf2672SBrett Creeley spin_lock_init(&vsi->arfs_lock); 52828bf2672SBrett Creeley 52928bf2672SBrett Creeley vsi->arfs_fltr_list = arfs_fltr_list; 53028bf2672SBrett Creeley 53128bf2672SBrett Creeley return; 53228bf2672SBrett Creeley 53328bf2672SBrett Creeley free_arfs_fltr_list: 53428bf2672SBrett Creeley kfree(arfs_fltr_list); 53528bf2672SBrett Creeley } 53628bf2672SBrett Creeley 53728bf2672SBrett Creeley /** 53828bf2672SBrett Creeley * ice_clear_arfs - clear the aRFS hash table and any memory used for aRFS 53928bf2672SBrett Creeley * @vsi: the VSI to be forwarded to 54028bf2672SBrett Creeley */ 54128bf2672SBrett Creeley void ice_clear_arfs(struct ice_vsi *vsi) 54228bf2672SBrett Creeley { 54328bf2672SBrett Creeley struct device *dev; 54428bf2672SBrett Creeley unsigned int i; 54528bf2672SBrett Creeley 54628bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF || !vsi->back || 54728bf2672SBrett Creeley !vsi->arfs_fltr_list) 54828bf2672SBrett Creeley return; 54928bf2672SBrett Creeley 55028bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back); 55128bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++) { 55228bf2672SBrett Creeley struct ice_arfs_entry *r; 55328bf2672SBrett Creeley struct hlist_node *n; 55428bf2672SBrett Creeley 55528bf2672SBrett Creeley spin_lock_bh(&vsi->arfs_lock); 55628bf2672SBrett Creeley hlist_for_each_entry_safe(r, n, &vsi->arfs_fltr_list[i], 55728bf2672SBrett Creeley list_entry) { 55828bf2672SBrett Creeley hlist_del(&r->list_entry); 55928bf2672SBrett Creeley devm_kfree(dev, r); 56028bf2672SBrett Creeley } 56128bf2672SBrett Creeley spin_unlock_bh(&vsi->arfs_lock); 56228bf2672SBrett Creeley } 56328bf2672SBrett Creeley 56428bf2672SBrett Creeley kfree(vsi->arfs_fltr_list); 56528bf2672SBrett Creeley vsi->arfs_fltr_list = NULL; 56628bf2672SBrett Creeley kfree(vsi->arfs_last_fltr_id); 56728bf2672SBrett Creeley vsi->arfs_last_fltr_id = NULL; 56828bf2672SBrett Creeley kfree(vsi->arfs_fltr_cntrs); 56928bf2672SBrett Creeley vsi->arfs_fltr_cntrs = NULL; 57028bf2672SBrett Creeley } 57128bf2672SBrett Creeley 57228bf2672SBrett Creeley /** 57328bf2672SBrett Creeley * ice_free_cpu_rx_rmap - free setup CPU reverse map 57428bf2672SBrett Creeley * @vsi: the VSI to be forwarded to 57528bf2672SBrett Creeley */ 57628bf2672SBrett Creeley void ice_free_cpu_rx_rmap(struct ice_vsi *vsi) 57728bf2672SBrett Creeley { 57828bf2672SBrett Creeley struct net_device *netdev; 57928bf2672SBrett Creeley 58028bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF || !vsi->arfs_fltr_list) 58128bf2672SBrett Creeley return; 58228bf2672SBrett Creeley 58328bf2672SBrett Creeley netdev = vsi->netdev; 584*1e23f076SAnirudh Venkataramanan if (!netdev || !netdev->rx_cpu_rmap) 58528bf2672SBrett Creeley return; 58628bf2672SBrett Creeley 58728bf2672SBrett Creeley free_irq_cpu_rmap(netdev->rx_cpu_rmap); 58828bf2672SBrett Creeley netdev->rx_cpu_rmap = NULL; 58928bf2672SBrett Creeley } 59028bf2672SBrett Creeley 59128bf2672SBrett Creeley /** 59228bf2672SBrett Creeley * ice_set_cpu_rx_rmap - setup CPU reverse map for each queue 59328bf2672SBrett Creeley * @vsi: the VSI to be forwarded to 59428bf2672SBrett Creeley */ 59528bf2672SBrett Creeley int ice_set_cpu_rx_rmap(struct ice_vsi *vsi) 59628bf2672SBrett Creeley { 59728bf2672SBrett Creeley struct net_device *netdev; 59828bf2672SBrett Creeley struct ice_pf *pf; 59928bf2672SBrett Creeley int base_idx, i; 60028bf2672SBrett Creeley 60128bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF) 60228bf2672SBrett Creeley return -EINVAL; 60328bf2672SBrett Creeley 60428bf2672SBrett Creeley pf = vsi->back; 60528bf2672SBrett Creeley netdev = vsi->netdev; 606*1e23f076SAnirudh Venkataramanan if (!pf || !netdev || !vsi->num_q_vectors) 60728bf2672SBrett Creeley return -EINVAL; 60828bf2672SBrett Creeley 60928bf2672SBrett Creeley netdev_dbg(netdev, "Setup CPU RMAP: vsi type 0x%x, ifname %s, q_vectors %d\n", 61028bf2672SBrett Creeley vsi->type, netdev->name, vsi->num_q_vectors); 61128bf2672SBrett Creeley 61228bf2672SBrett Creeley netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(vsi->num_q_vectors); 61328bf2672SBrett Creeley if (unlikely(!netdev->rx_cpu_rmap)) 61428bf2672SBrett Creeley return -EINVAL; 61528bf2672SBrett Creeley 61628bf2672SBrett Creeley base_idx = vsi->base_vector; 61728bf2672SBrett Creeley for (i = 0; i < vsi->num_q_vectors; i++) 61828bf2672SBrett Creeley if (irq_cpu_rmap_add(netdev->rx_cpu_rmap, 61928bf2672SBrett Creeley pf->msix_entries[base_idx + i].vector)) { 62028bf2672SBrett Creeley ice_free_cpu_rx_rmap(vsi); 62128bf2672SBrett Creeley return -EINVAL; 62228bf2672SBrett Creeley } 62328bf2672SBrett Creeley 62428bf2672SBrett Creeley return 0; 62528bf2672SBrett Creeley } 62628bf2672SBrett Creeley 62728bf2672SBrett Creeley /** 62828bf2672SBrett Creeley * ice_remove_arfs - remove/clear all aRFS resources 62928bf2672SBrett Creeley * @pf: device private structure 63028bf2672SBrett Creeley */ 63128bf2672SBrett Creeley void ice_remove_arfs(struct ice_pf *pf) 63228bf2672SBrett Creeley { 63328bf2672SBrett Creeley struct ice_vsi *pf_vsi; 63428bf2672SBrett Creeley 63528bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf); 63628bf2672SBrett Creeley if (!pf_vsi) 63728bf2672SBrett Creeley return; 63828bf2672SBrett Creeley 63928bf2672SBrett Creeley ice_free_cpu_rx_rmap(pf_vsi); 64028bf2672SBrett Creeley ice_clear_arfs(pf_vsi); 64128bf2672SBrett Creeley } 64228bf2672SBrett Creeley 64328bf2672SBrett Creeley /** 64428bf2672SBrett Creeley * ice_rebuild_arfs - remove/clear all aRFS resources and rebuild after reset 64528bf2672SBrett Creeley * @pf: device private structure 64628bf2672SBrett Creeley */ 64728bf2672SBrett Creeley void ice_rebuild_arfs(struct ice_pf *pf) 64828bf2672SBrett Creeley { 64928bf2672SBrett Creeley struct ice_vsi *pf_vsi; 65028bf2672SBrett Creeley 65128bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf); 65228bf2672SBrett Creeley if (!pf_vsi) 65328bf2672SBrett Creeley return; 65428bf2672SBrett Creeley 65528bf2672SBrett Creeley ice_remove_arfs(pf); 65628bf2672SBrett Creeley if (ice_set_cpu_rx_rmap(pf_vsi)) { 65728bf2672SBrett Creeley dev_err(ice_pf_to_dev(pf), "Failed to rebuild aRFS\n"); 65828bf2672SBrett Creeley return; 65928bf2672SBrett Creeley } 66028bf2672SBrett Creeley ice_init_arfs(pf_vsi); 66128bf2672SBrett Creeley } 662