128bf2672SBrett Creeley // SPDX-License-Identifier: GPL-2.0
228bf2672SBrett Creeley /* Copyright (C) 2018-2020, Intel Corporation. */
328bf2672SBrett Creeley
428bf2672SBrett Creeley #include "ice.h"
528bf2672SBrett Creeley
628bf2672SBrett Creeley /**
728bf2672SBrett Creeley * ice_is_arfs_active - helper to check is aRFS is active
828bf2672SBrett Creeley * @vsi: VSI to check
928bf2672SBrett Creeley */
ice_is_arfs_active(struct ice_vsi * vsi)1028bf2672SBrett Creeley static bool ice_is_arfs_active(struct ice_vsi *vsi)
1128bf2672SBrett Creeley {
1228bf2672SBrett Creeley return !!vsi->arfs_fltr_list;
1328bf2672SBrett Creeley }
1428bf2672SBrett Creeley
1528bf2672SBrett Creeley /**
1628bf2672SBrett Creeley * ice_is_arfs_using_perfect_flow - check if aRFS has active perfect filters
1728bf2672SBrett Creeley * @hw: pointer to the HW structure
1828bf2672SBrett Creeley * @flow_type: flow type as Flow Director understands it
1928bf2672SBrett Creeley *
2028bf2672SBrett Creeley * Flow Director will query this function to see if aRFS is currently using
2128bf2672SBrett Creeley * the specified flow_type for perfect (4-tuple) filters.
2228bf2672SBrett Creeley */
2328bf2672SBrett Creeley bool
ice_is_arfs_using_perfect_flow(struct ice_hw * hw,enum ice_fltr_ptype flow_type)2428bf2672SBrett Creeley ice_is_arfs_using_perfect_flow(struct ice_hw *hw, enum ice_fltr_ptype flow_type)
2528bf2672SBrett Creeley {
2628bf2672SBrett Creeley struct ice_arfs_active_fltr_cntrs *arfs_fltr_cntrs;
2728bf2672SBrett Creeley struct ice_pf *pf = hw->back;
2828bf2672SBrett Creeley struct ice_vsi *vsi;
2928bf2672SBrett Creeley
3028bf2672SBrett Creeley vsi = ice_get_main_vsi(pf);
3128bf2672SBrett Creeley if (!vsi)
3228bf2672SBrett Creeley return false;
3328bf2672SBrett Creeley
3428bf2672SBrett Creeley arfs_fltr_cntrs = vsi->arfs_fltr_cntrs;
3528bf2672SBrett Creeley
3628bf2672SBrett Creeley /* active counters can be updated by multiple CPUs */
3728bf2672SBrett Creeley smp_mb__before_atomic();
3828bf2672SBrett Creeley switch (flow_type) {
3928bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_UDP:
4028bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_udpv4_cnt) > 0;
4128bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_UDP:
4228bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_udpv6_cnt) > 0;
4328bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_TCP:
4428bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_tcpv4_cnt) > 0;
4528bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_TCP:
4628bf2672SBrett Creeley return atomic_read(&arfs_fltr_cntrs->active_tcpv6_cnt) > 0;
4728bf2672SBrett Creeley default:
4828bf2672SBrett Creeley return false;
4928bf2672SBrett Creeley }
5028bf2672SBrett Creeley }
5128bf2672SBrett Creeley
5228bf2672SBrett Creeley /**
5328bf2672SBrett Creeley * ice_arfs_update_active_fltr_cntrs - update active filter counters for aRFS
5428bf2672SBrett Creeley * @vsi: VSI that aRFS is active on
5528bf2672SBrett Creeley * @entry: aRFS entry used to change counters
5628bf2672SBrett Creeley * @add: true to increment counter, false to decrement
5728bf2672SBrett Creeley */
5828bf2672SBrett Creeley static void
ice_arfs_update_active_fltr_cntrs(struct ice_vsi * vsi,struct ice_arfs_entry * entry,bool add)5928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(struct ice_vsi *vsi,
6028bf2672SBrett Creeley struct ice_arfs_entry *entry, bool add)
6128bf2672SBrett Creeley {
6228bf2672SBrett Creeley struct ice_arfs_active_fltr_cntrs *fltr_cntrs = vsi->arfs_fltr_cntrs;
6328bf2672SBrett Creeley
6428bf2672SBrett Creeley switch (entry->fltr_info.flow_type) {
6528bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_TCP:
6628bf2672SBrett Creeley if (add)
6728bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_tcpv4_cnt);
6828bf2672SBrett Creeley else
6928bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_tcpv4_cnt);
7028bf2672SBrett Creeley break;
7128bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_TCP:
7228bf2672SBrett Creeley if (add)
7328bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_tcpv6_cnt);
7428bf2672SBrett Creeley else
7528bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_tcpv6_cnt);
7628bf2672SBrett Creeley break;
7728bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV4_UDP:
7828bf2672SBrett Creeley if (add)
7928bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_udpv4_cnt);
8028bf2672SBrett Creeley else
8128bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_udpv4_cnt);
8228bf2672SBrett Creeley break;
8328bf2672SBrett Creeley case ICE_FLTR_PTYPE_NONF_IPV6_UDP:
8428bf2672SBrett Creeley if (add)
8528bf2672SBrett Creeley atomic_inc(&fltr_cntrs->active_udpv6_cnt);
8628bf2672SBrett Creeley else
8728bf2672SBrett Creeley atomic_dec(&fltr_cntrs->active_udpv6_cnt);
8828bf2672SBrett Creeley break;
8928bf2672SBrett Creeley default:
9028bf2672SBrett Creeley dev_err(ice_pf_to_dev(vsi->back), "aRFS: Failed to update filter counters, invalid filter type %d\n",
9128bf2672SBrett Creeley entry->fltr_info.flow_type);
9228bf2672SBrett Creeley }
9328bf2672SBrett Creeley }
9428bf2672SBrett Creeley
9528bf2672SBrett Creeley /**
9628bf2672SBrett Creeley * ice_arfs_del_flow_rules - delete the rules passed in from HW
9728bf2672SBrett Creeley * @vsi: VSI for the flow rules that need to be deleted
9828bf2672SBrett Creeley * @del_list_head: head of the list of ice_arfs_entry(s) for rule deletion
9928bf2672SBrett Creeley *
10028bf2672SBrett Creeley * Loop through the delete list passed in and remove the rules from HW. After
10128bf2672SBrett Creeley * each rule is deleted, disconnect and free the ice_arfs_entry because it is no
10228bf2672SBrett Creeley * longer being referenced by the aRFS hash table.
10328bf2672SBrett Creeley */
10428bf2672SBrett Creeley static void
ice_arfs_del_flow_rules(struct ice_vsi * vsi,struct hlist_head * del_list_head)10528bf2672SBrett Creeley ice_arfs_del_flow_rules(struct ice_vsi *vsi, struct hlist_head *del_list_head)
10628bf2672SBrett Creeley {
10728bf2672SBrett Creeley struct ice_arfs_entry *e;
10828bf2672SBrett Creeley struct hlist_node *n;
10928bf2672SBrett Creeley struct device *dev;
11028bf2672SBrett Creeley
11128bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back);
11228bf2672SBrett Creeley
11328bf2672SBrett Creeley hlist_for_each_entry_safe(e, n, del_list_head, list_entry) {
11428bf2672SBrett Creeley int result;
11528bf2672SBrett Creeley
11628bf2672SBrett Creeley result = ice_fdir_write_fltr(vsi->back, &e->fltr_info, false,
11728bf2672SBrett Creeley false);
11828bf2672SBrett Creeley if (!result)
11928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, e, false);
12028bf2672SBrett Creeley else
12128bf2672SBrett Creeley dev_dbg(dev, "Unable to delete aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n",
12228bf2672SBrett Creeley result, e->fltr_state, e->fltr_info.fltr_id,
12328bf2672SBrett Creeley e->flow_id, e->fltr_info.q_index);
12428bf2672SBrett Creeley
12528bf2672SBrett Creeley /* The aRFS hash table is no longer referencing this entry */
12628bf2672SBrett Creeley hlist_del(&e->list_entry);
12728bf2672SBrett Creeley devm_kfree(dev, e);
12828bf2672SBrett Creeley }
12928bf2672SBrett Creeley }
13028bf2672SBrett Creeley
13128bf2672SBrett Creeley /**
13228bf2672SBrett Creeley * ice_arfs_add_flow_rules - add the rules passed in from HW
13328bf2672SBrett Creeley * @vsi: VSI for the flow rules that need to be added
13428bf2672SBrett Creeley * @add_list_head: head of the list of ice_arfs_entry_ptr(s) for rule addition
13528bf2672SBrett Creeley *
13628bf2672SBrett Creeley * Loop through the add list passed in and remove the rules from HW. After each
13728bf2672SBrett Creeley * rule is added, disconnect and free the ice_arfs_entry_ptr node. Don't free
13828bf2672SBrett Creeley * the ice_arfs_entry(s) because they are still being referenced in the aRFS
13928bf2672SBrett Creeley * hash table.
14028bf2672SBrett Creeley */
14128bf2672SBrett Creeley static void
ice_arfs_add_flow_rules(struct ice_vsi * vsi,struct hlist_head * add_list_head)14228bf2672SBrett Creeley ice_arfs_add_flow_rules(struct ice_vsi *vsi, struct hlist_head *add_list_head)
14328bf2672SBrett Creeley {
14428bf2672SBrett Creeley struct ice_arfs_entry_ptr *ep;
14528bf2672SBrett Creeley struct hlist_node *n;
14628bf2672SBrett Creeley struct device *dev;
14728bf2672SBrett Creeley
14828bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back);
14928bf2672SBrett Creeley
15028bf2672SBrett Creeley hlist_for_each_entry_safe(ep, n, add_list_head, list_entry) {
15128bf2672SBrett Creeley int result;
15228bf2672SBrett Creeley
15328bf2672SBrett Creeley result = ice_fdir_write_fltr(vsi->back,
15428bf2672SBrett Creeley &ep->arfs_entry->fltr_info, true,
15528bf2672SBrett Creeley false);
15628bf2672SBrett Creeley if (!result)
15728bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, ep->arfs_entry,
15828bf2672SBrett Creeley true);
15928bf2672SBrett Creeley else
16028bf2672SBrett Creeley dev_dbg(dev, "Unable to add aRFS entry, err %d fltr_state %d fltr_id %d flow_id %d Q %d\n",
16128bf2672SBrett Creeley result, ep->arfs_entry->fltr_state,
16228bf2672SBrett Creeley ep->arfs_entry->fltr_info.fltr_id,
16328bf2672SBrett Creeley ep->arfs_entry->flow_id,
16428bf2672SBrett Creeley ep->arfs_entry->fltr_info.q_index);
16528bf2672SBrett Creeley
16628bf2672SBrett Creeley hlist_del(&ep->list_entry);
16728bf2672SBrett Creeley devm_kfree(dev, ep);
16828bf2672SBrett Creeley }
16928bf2672SBrett Creeley }
17028bf2672SBrett Creeley
17128bf2672SBrett Creeley /**
17228bf2672SBrett Creeley * ice_arfs_is_flow_expired - check if the aRFS entry has expired
17328bf2672SBrett Creeley * @vsi: VSI containing the aRFS entry
17428bf2672SBrett Creeley * @arfs_entry: aRFS entry that's being checked for expiration
17528bf2672SBrett Creeley *
17628bf2672SBrett Creeley * Return true if the flow has expired, else false. This function should be used
17728bf2672SBrett Creeley * to determine whether or not an aRFS entry should be removed from the hardware
17828bf2672SBrett Creeley * and software structures.
17928bf2672SBrett Creeley */
18028bf2672SBrett Creeley static bool
ice_arfs_is_flow_expired(struct ice_vsi * vsi,struct ice_arfs_entry * arfs_entry)18128bf2672SBrett Creeley ice_arfs_is_flow_expired(struct ice_vsi *vsi, struct ice_arfs_entry *arfs_entry)
18228bf2672SBrett Creeley {
18328bf2672SBrett Creeley #define ICE_ARFS_TIME_DELTA_EXPIRATION msecs_to_jiffies(5000)
18428bf2672SBrett Creeley if (rps_may_expire_flow(vsi->netdev, arfs_entry->fltr_info.q_index,
18528bf2672SBrett Creeley arfs_entry->flow_id,
18628bf2672SBrett Creeley arfs_entry->fltr_info.fltr_id))
18728bf2672SBrett Creeley return true;
18828bf2672SBrett Creeley
18928bf2672SBrett Creeley /* expiration timer only used for UDP filters */
19028bf2672SBrett Creeley if (arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV4_UDP &&
19128bf2672SBrett Creeley arfs_entry->fltr_info.flow_type != ICE_FLTR_PTYPE_NONF_IPV6_UDP)
19228bf2672SBrett Creeley return false;
19328bf2672SBrett Creeley
19428bf2672SBrett Creeley return time_in_range64(arfs_entry->time_activated +
19528bf2672SBrett Creeley ICE_ARFS_TIME_DELTA_EXPIRATION,
19628bf2672SBrett Creeley arfs_entry->time_activated, get_jiffies_64());
19728bf2672SBrett Creeley }
19828bf2672SBrett Creeley
19928bf2672SBrett Creeley /**
20028bf2672SBrett Creeley * ice_arfs_update_flow_rules - add/delete aRFS rules in HW
20128bf2672SBrett Creeley * @vsi: the VSI to be forwarded to
20228bf2672SBrett Creeley * @idx: index into the table of aRFS filter lists. Obtained from skb->hash
20328bf2672SBrett Creeley * @add_list: list to populate with filters to be added to Flow Director
20428bf2672SBrett Creeley * @del_list: list to populate with filters to be deleted from Flow Director
20528bf2672SBrett Creeley *
20628bf2672SBrett Creeley * Iterate over the hlist at the index given in the aRFS hash table and
20728bf2672SBrett Creeley * determine if there are any aRFS entries that need to be either added or
20828bf2672SBrett Creeley * deleted in the HW. If the aRFS entry is marked as ICE_ARFS_INACTIVE the
20928bf2672SBrett Creeley * filter needs to be added to HW, else if it's marked as ICE_ARFS_ACTIVE and
21028bf2672SBrett Creeley * the flow has expired delete the filter from HW. The caller of this function
21128bf2672SBrett Creeley * is expected to add/delete rules on the add_list/del_list respectively.
21228bf2672SBrett Creeley */
21328bf2672SBrett Creeley static void
ice_arfs_update_flow_rules(struct ice_vsi * vsi,u16 idx,struct hlist_head * add_list,struct hlist_head * del_list)21428bf2672SBrett Creeley ice_arfs_update_flow_rules(struct ice_vsi *vsi, u16 idx,
21528bf2672SBrett Creeley struct hlist_head *add_list,
21628bf2672SBrett Creeley struct hlist_head *del_list)
21728bf2672SBrett Creeley {
21828bf2672SBrett Creeley struct ice_arfs_entry *e;
21928bf2672SBrett Creeley struct hlist_node *n;
22028bf2672SBrett Creeley struct device *dev;
22128bf2672SBrett Creeley
22228bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back);
22328bf2672SBrett Creeley
22428bf2672SBrett Creeley /* go through the aRFS hlist at this idx and check for needed updates */
22528bf2672SBrett Creeley hlist_for_each_entry_safe(e, n, &vsi->arfs_fltr_list[idx], list_entry)
22628bf2672SBrett Creeley /* check if filter needs to be added to HW */
22728bf2672SBrett Creeley if (e->fltr_state == ICE_ARFS_INACTIVE) {
22828bf2672SBrett Creeley enum ice_fltr_ptype flow_type = e->fltr_info.flow_type;
22928bf2672SBrett Creeley struct ice_arfs_entry_ptr *ep =
23028bf2672SBrett Creeley devm_kzalloc(dev, sizeof(*ep), GFP_ATOMIC);
23128bf2672SBrett Creeley
23228bf2672SBrett Creeley if (!ep)
23328bf2672SBrett Creeley continue;
23428bf2672SBrett Creeley INIT_HLIST_NODE(&ep->list_entry);
23528bf2672SBrett Creeley /* reference aRFS entry to add HW filter */
23628bf2672SBrett Creeley ep->arfs_entry = e;
23728bf2672SBrett Creeley hlist_add_head(&ep->list_entry, add_list);
23828bf2672SBrett Creeley e->fltr_state = ICE_ARFS_ACTIVE;
23928bf2672SBrett Creeley /* expiration timer only used for UDP flows */
24028bf2672SBrett Creeley if (flow_type == ICE_FLTR_PTYPE_NONF_IPV4_UDP ||
24128bf2672SBrett Creeley flow_type == ICE_FLTR_PTYPE_NONF_IPV6_UDP)
24228bf2672SBrett Creeley e->time_activated = get_jiffies_64();
24328bf2672SBrett Creeley } else if (e->fltr_state == ICE_ARFS_ACTIVE) {
24428bf2672SBrett Creeley /* check if filter needs to be removed from HW */
24528bf2672SBrett Creeley if (ice_arfs_is_flow_expired(vsi, e)) {
24628bf2672SBrett Creeley /* remove aRFS entry from hash table for delete
24728bf2672SBrett Creeley * and to prevent referencing it the next time
24828bf2672SBrett Creeley * through this hlist index
24928bf2672SBrett Creeley */
25028bf2672SBrett Creeley hlist_del(&e->list_entry);
25128bf2672SBrett Creeley e->fltr_state = ICE_ARFS_TODEL;
25228bf2672SBrett Creeley /* save reference to aRFS entry for delete */
25328bf2672SBrett Creeley hlist_add_head(&e->list_entry, del_list);
25428bf2672SBrett Creeley }
25528bf2672SBrett Creeley }
25628bf2672SBrett Creeley }
25728bf2672SBrett Creeley
25828bf2672SBrett Creeley /**
25928bf2672SBrett Creeley * ice_sync_arfs_fltrs - update all aRFS filters
26028bf2672SBrett Creeley * @pf: board private structure
26128bf2672SBrett Creeley */
ice_sync_arfs_fltrs(struct ice_pf * pf)26228bf2672SBrett Creeley void ice_sync_arfs_fltrs(struct ice_pf *pf)
26328bf2672SBrett Creeley {
26428bf2672SBrett Creeley HLIST_HEAD(tmp_del_list);
26528bf2672SBrett Creeley HLIST_HEAD(tmp_add_list);
26628bf2672SBrett Creeley struct ice_vsi *pf_vsi;
26728bf2672SBrett Creeley unsigned int i;
26828bf2672SBrett Creeley
26928bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf);
27028bf2672SBrett Creeley if (!pf_vsi)
27128bf2672SBrett Creeley return;
27228bf2672SBrett Creeley
27328bf2672SBrett Creeley if (!ice_is_arfs_active(pf_vsi))
27428bf2672SBrett Creeley return;
27528bf2672SBrett Creeley
27628bf2672SBrett Creeley spin_lock_bh(&pf_vsi->arfs_lock);
27728bf2672SBrett Creeley /* Once we process aRFS for the PF VSI get out */
27828bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++)
27928bf2672SBrett Creeley ice_arfs_update_flow_rules(pf_vsi, i, &tmp_add_list,
28028bf2672SBrett Creeley &tmp_del_list);
28128bf2672SBrett Creeley spin_unlock_bh(&pf_vsi->arfs_lock);
28228bf2672SBrett Creeley
28328bf2672SBrett Creeley /* use list of ice_arfs_entry(s) for delete */
28428bf2672SBrett Creeley ice_arfs_del_flow_rules(pf_vsi, &tmp_del_list);
28528bf2672SBrett Creeley
28628bf2672SBrett Creeley /* use list of ice_arfs_entry_ptr(s) for add */
28728bf2672SBrett Creeley ice_arfs_add_flow_rules(pf_vsi, &tmp_add_list);
28828bf2672SBrett Creeley }
28928bf2672SBrett Creeley
29028bf2672SBrett Creeley /**
29128bf2672SBrett Creeley * ice_arfs_build_entry - builds an aRFS entry based on input
29228bf2672SBrett Creeley * @vsi: destination VSI for this flow
29328bf2672SBrett Creeley * @fk: flow dissector keys for creating the tuple
29428bf2672SBrett Creeley * @rxq_idx: Rx queue to steer this flow to
29528bf2672SBrett Creeley * @flow_id: passed down from the stack and saved for flow expiration
29628bf2672SBrett Creeley *
29728bf2672SBrett Creeley * returns an aRFS entry on success and NULL on failure
29828bf2672SBrett Creeley */
29928bf2672SBrett Creeley static struct ice_arfs_entry *
ice_arfs_build_entry(struct ice_vsi * vsi,const struct flow_keys * fk,u16 rxq_idx,u32 flow_id)30028bf2672SBrett Creeley ice_arfs_build_entry(struct ice_vsi *vsi, const struct flow_keys *fk,
30128bf2672SBrett Creeley u16 rxq_idx, u32 flow_id)
30228bf2672SBrett Creeley {
30328bf2672SBrett Creeley struct ice_arfs_entry *arfs_entry;
30428bf2672SBrett Creeley struct ice_fdir_fltr *fltr_info;
30528bf2672SBrett Creeley u8 ip_proto;
30628bf2672SBrett Creeley
30728bf2672SBrett Creeley arfs_entry = devm_kzalloc(ice_pf_to_dev(vsi->back),
30828bf2672SBrett Creeley sizeof(*arfs_entry),
30928bf2672SBrett Creeley GFP_ATOMIC | __GFP_NOWARN);
31028bf2672SBrett Creeley if (!arfs_entry)
31128bf2672SBrett Creeley return NULL;
31228bf2672SBrett Creeley
31328bf2672SBrett Creeley fltr_info = &arfs_entry->fltr_info;
31428bf2672SBrett Creeley fltr_info->q_index = rxq_idx;
31528bf2672SBrett Creeley fltr_info->dest_ctl = ICE_FLTR_PRGM_DESC_DEST_DIRECT_PKT_QINDEX;
31628bf2672SBrett Creeley fltr_info->dest_vsi = vsi->idx;
31728bf2672SBrett Creeley ip_proto = fk->basic.ip_proto;
31828bf2672SBrett Creeley
31928bf2672SBrett Creeley if (fk->basic.n_proto == htons(ETH_P_IP)) {
32028bf2672SBrett Creeley fltr_info->ip.v4.proto = ip_proto;
32128bf2672SBrett Creeley fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ?
32228bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV4_TCP :
32328bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV4_UDP;
32428bf2672SBrett Creeley fltr_info->ip.v4.src_ip = fk->addrs.v4addrs.src;
32528bf2672SBrett Creeley fltr_info->ip.v4.dst_ip = fk->addrs.v4addrs.dst;
32628bf2672SBrett Creeley fltr_info->ip.v4.src_port = fk->ports.src;
32728bf2672SBrett Creeley fltr_info->ip.v4.dst_port = fk->ports.dst;
32828bf2672SBrett Creeley } else { /* ETH_P_IPV6 */
32928bf2672SBrett Creeley fltr_info->ip.v6.proto = ip_proto;
33028bf2672SBrett Creeley fltr_info->flow_type = (ip_proto == IPPROTO_TCP) ?
33128bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV6_TCP :
33228bf2672SBrett Creeley ICE_FLTR_PTYPE_NONF_IPV6_UDP;
33328bf2672SBrett Creeley memcpy(&fltr_info->ip.v6.src_ip, &fk->addrs.v6addrs.src,
33428bf2672SBrett Creeley sizeof(struct in6_addr));
33528bf2672SBrett Creeley memcpy(&fltr_info->ip.v6.dst_ip, &fk->addrs.v6addrs.dst,
33628bf2672SBrett Creeley sizeof(struct in6_addr));
33728bf2672SBrett Creeley fltr_info->ip.v6.src_port = fk->ports.src;
33828bf2672SBrett Creeley fltr_info->ip.v6.dst_port = fk->ports.dst;
33928bf2672SBrett Creeley }
34028bf2672SBrett Creeley
34128bf2672SBrett Creeley arfs_entry->flow_id = flow_id;
34228bf2672SBrett Creeley fltr_info->fltr_id =
34328bf2672SBrett Creeley atomic_inc_return(vsi->arfs_last_fltr_id) % RPS_NO_FILTER;
34428bf2672SBrett Creeley
34528bf2672SBrett Creeley return arfs_entry;
34628bf2672SBrett Creeley }
34728bf2672SBrett Creeley
34828bf2672SBrett Creeley /**
34928bf2672SBrett Creeley * ice_arfs_is_perfect_flow_set - Check to see if perfect flow is set
35028bf2672SBrett Creeley * @hw: pointer to HW structure
35128bf2672SBrett Creeley * @l3_proto: ETH_P_IP or ETH_P_IPV6 in network order
35228bf2672SBrett Creeley * @l4_proto: IPPROTO_UDP or IPPROTO_TCP
35328bf2672SBrett Creeley *
35428bf2672SBrett Creeley * We only support perfect (4-tuple) filters for aRFS. This function allows aRFS
35528bf2672SBrett Creeley * to check if perfect (4-tuple) flow rules are currently in place by Flow
35628bf2672SBrett Creeley * Director.
35728bf2672SBrett Creeley */
35828bf2672SBrett Creeley static bool
ice_arfs_is_perfect_flow_set(struct ice_hw * hw,__be16 l3_proto,u8 l4_proto)35928bf2672SBrett Creeley ice_arfs_is_perfect_flow_set(struct ice_hw *hw, __be16 l3_proto, u8 l4_proto)
36028bf2672SBrett Creeley {
36128bf2672SBrett Creeley unsigned long *perfect_fltr = hw->fdir_perfect_fltr;
36228bf2672SBrett Creeley
36328bf2672SBrett Creeley /* advanced Flow Director disabled, perfect filters always supported */
36428bf2672SBrett Creeley if (!perfect_fltr)
36528bf2672SBrett Creeley return true;
36628bf2672SBrett Creeley
36728bf2672SBrett Creeley if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_UDP)
36828bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_UDP, perfect_fltr);
36928bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IP) && l4_proto == IPPROTO_TCP)
37028bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV4_TCP, perfect_fltr);
37128bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_UDP)
37228bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_UDP, perfect_fltr);
37328bf2672SBrett Creeley else if (l3_proto == htons(ETH_P_IPV6) && l4_proto == IPPROTO_TCP)
37428bf2672SBrett Creeley return test_bit(ICE_FLTR_PTYPE_NONF_IPV6_TCP, perfect_fltr);
37528bf2672SBrett Creeley
37628bf2672SBrett Creeley return false;
37728bf2672SBrett Creeley }
37828bf2672SBrett Creeley
37928bf2672SBrett Creeley /**
38028bf2672SBrett Creeley * ice_rx_flow_steer - steer the Rx flow to where application is being run
38128bf2672SBrett Creeley * @netdev: ptr to the netdev being adjusted
38228bf2672SBrett Creeley * @skb: buffer with required header information
38328bf2672SBrett Creeley * @rxq_idx: queue to which the flow needs to move
38428bf2672SBrett Creeley * @flow_id: flow identifier provided by the netdev
38528bf2672SBrett Creeley *
38628bf2672SBrett Creeley * Based on the skb, rxq_idx, and flow_id passed in add/update an entry in the
38728bf2672SBrett Creeley * aRFS hash table. Iterate over one of the hlists in the aRFS hash table and
38828bf2672SBrett Creeley * if the flow_id already exists in the hash table but the rxq_idx has changed
38928bf2672SBrett Creeley * mark the entry as ICE_ARFS_INACTIVE so it can get updated in HW, else
39028bf2672SBrett Creeley * if the entry is marked as ICE_ARFS_TODEL delete it from the aRFS hash table.
39128bf2672SBrett Creeley * If neither of the previous conditions are true then add a new entry in the
39228bf2672SBrett Creeley * aRFS hash table, which gets set to ICE_ARFS_INACTIVE by default so it can be
39328bf2672SBrett Creeley * added to HW.
39428bf2672SBrett Creeley */
39528bf2672SBrett Creeley int
ice_rx_flow_steer(struct net_device * netdev,const struct sk_buff * skb,u16 rxq_idx,u32 flow_id)39628bf2672SBrett Creeley ice_rx_flow_steer(struct net_device *netdev, const struct sk_buff *skb,
39728bf2672SBrett Creeley u16 rxq_idx, u32 flow_id)
39828bf2672SBrett Creeley {
39928bf2672SBrett Creeley struct ice_netdev_priv *np = netdev_priv(netdev);
40028bf2672SBrett Creeley struct ice_arfs_entry *arfs_entry;
40128bf2672SBrett Creeley struct ice_vsi *vsi = np->vsi;
40228bf2672SBrett Creeley struct flow_keys fk;
40328bf2672SBrett Creeley struct ice_pf *pf;
40428bf2672SBrett Creeley __be16 n_proto;
40528bf2672SBrett Creeley u8 ip_proto;
40628bf2672SBrett Creeley u16 idx;
40728bf2672SBrett Creeley int ret;
40828bf2672SBrett Creeley
40928bf2672SBrett Creeley /* failed to allocate memory for aRFS so don't crash */
41028bf2672SBrett Creeley if (unlikely(!vsi->arfs_fltr_list))
41128bf2672SBrett Creeley return -ENODEV;
41228bf2672SBrett Creeley
41328bf2672SBrett Creeley pf = vsi->back;
41428bf2672SBrett Creeley
41528bf2672SBrett Creeley if (skb->encapsulation)
41628bf2672SBrett Creeley return -EPROTONOSUPPORT;
41728bf2672SBrett Creeley
41828bf2672SBrett Creeley if (!skb_flow_dissect_flow_keys(skb, &fk, 0))
41928bf2672SBrett Creeley return -EPROTONOSUPPORT;
42028bf2672SBrett Creeley
42128bf2672SBrett Creeley n_proto = fk.basic.n_proto;
42228bf2672SBrett Creeley /* Support only IPV4 and IPV6 */
42328bf2672SBrett Creeley if ((n_proto == htons(ETH_P_IP) && !ip_is_fragment(ip_hdr(skb))) ||
42428bf2672SBrett Creeley n_proto == htons(ETH_P_IPV6))
42528bf2672SBrett Creeley ip_proto = fk.basic.ip_proto;
42628bf2672SBrett Creeley else
42728bf2672SBrett Creeley return -EPROTONOSUPPORT;
42828bf2672SBrett Creeley
42928bf2672SBrett Creeley /* Support only TCP and UDP */
43028bf2672SBrett Creeley if (ip_proto != IPPROTO_TCP && ip_proto != IPPROTO_UDP)
43128bf2672SBrett Creeley return -EPROTONOSUPPORT;
43228bf2672SBrett Creeley
43328bf2672SBrett Creeley /* only support 4-tuple filters for aRFS */
43428bf2672SBrett Creeley if (!ice_arfs_is_perfect_flow_set(&pf->hw, n_proto, ip_proto))
43528bf2672SBrett Creeley return -EOPNOTSUPP;
43628bf2672SBrett Creeley
43728bf2672SBrett Creeley /* choose the aRFS list bucket based on skb hash */
43828bf2672SBrett Creeley idx = skb_get_hash_raw(skb) & ICE_ARFS_LST_MASK;
43928bf2672SBrett Creeley /* search for entry in the bucket */
44028bf2672SBrett Creeley spin_lock_bh(&vsi->arfs_lock);
44128bf2672SBrett Creeley hlist_for_each_entry(arfs_entry, &vsi->arfs_fltr_list[idx],
44228bf2672SBrett Creeley list_entry) {
44328bf2672SBrett Creeley struct ice_fdir_fltr *fltr_info;
44428bf2672SBrett Creeley
44528bf2672SBrett Creeley /* keep searching for the already existing arfs_entry flow */
44628bf2672SBrett Creeley if (arfs_entry->flow_id != flow_id)
44728bf2672SBrett Creeley continue;
44828bf2672SBrett Creeley
44928bf2672SBrett Creeley fltr_info = &arfs_entry->fltr_info;
45028bf2672SBrett Creeley ret = fltr_info->fltr_id;
45128bf2672SBrett Creeley
45228bf2672SBrett Creeley if (fltr_info->q_index == rxq_idx ||
45328bf2672SBrett Creeley arfs_entry->fltr_state != ICE_ARFS_ACTIVE)
45428bf2672SBrett Creeley goto out;
45528bf2672SBrett Creeley
45628bf2672SBrett Creeley /* update the queue to forward to on an already existing flow */
45728bf2672SBrett Creeley fltr_info->q_index = rxq_idx;
45828bf2672SBrett Creeley arfs_entry->fltr_state = ICE_ARFS_INACTIVE;
45928bf2672SBrett Creeley ice_arfs_update_active_fltr_cntrs(vsi, arfs_entry, false);
46028bf2672SBrett Creeley goto out_schedule_service_task;
46128bf2672SBrett Creeley }
46228bf2672SBrett Creeley
46328bf2672SBrett Creeley arfs_entry = ice_arfs_build_entry(vsi, &fk, rxq_idx, flow_id);
46428bf2672SBrett Creeley if (!arfs_entry) {
46528bf2672SBrett Creeley ret = -ENOMEM;
46628bf2672SBrett Creeley goto out;
46728bf2672SBrett Creeley }
46828bf2672SBrett Creeley
46928bf2672SBrett Creeley ret = arfs_entry->fltr_info.fltr_id;
47028bf2672SBrett Creeley INIT_HLIST_NODE(&arfs_entry->list_entry);
47128bf2672SBrett Creeley hlist_add_head(&arfs_entry->list_entry, &vsi->arfs_fltr_list[idx]);
47228bf2672SBrett Creeley out_schedule_service_task:
47328bf2672SBrett Creeley ice_service_task_schedule(pf);
47428bf2672SBrett Creeley out:
47528bf2672SBrett Creeley spin_unlock_bh(&vsi->arfs_lock);
47628bf2672SBrett Creeley return ret;
47728bf2672SBrett Creeley }
47828bf2672SBrett Creeley
47928bf2672SBrett Creeley /**
48028bf2672SBrett Creeley * ice_init_arfs_cntrs - initialize aRFS counter values
48128bf2672SBrett Creeley * @vsi: VSI that aRFS counters need to be initialized on
48228bf2672SBrett Creeley */
ice_init_arfs_cntrs(struct ice_vsi * vsi)48328bf2672SBrett Creeley static int ice_init_arfs_cntrs(struct ice_vsi *vsi)
48428bf2672SBrett Creeley {
48528bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF)
48628bf2672SBrett Creeley return -EINVAL;
48728bf2672SBrett Creeley
48828bf2672SBrett Creeley vsi->arfs_fltr_cntrs = kzalloc(sizeof(*vsi->arfs_fltr_cntrs),
48928bf2672SBrett Creeley GFP_KERNEL);
49028bf2672SBrett Creeley if (!vsi->arfs_fltr_cntrs)
49128bf2672SBrett Creeley return -ENOMEM;
49228bf2672SBrett Creeley
49328bf2672SBrett Creeley vsi->arfs_last_fltr_id = kzalloc(sizeof(*vsi->arfs_last_fltr_id),
49428bf2672SBrett Creeley GFP_KERNEL);
49528bf2672SBrett Creeley if (!vsi->arfs_last_fltr_id) {
49628bf2672SBrett Creeley kfree(vsi->arfs_fltr_cntrs);
49728bf2672SBrett Creeley vsi->arfs_fltr_cntrs = NULL;
49828bf2672SBrett Creeley return -ENOMEM;
49928bf2672SBrett Creeley }
50028bf2672SBrett Creeley
50128bf2672SBrett Creeley return 0;
50228bf2672SBrett Creeley }
50328bf2672SBrett Creeley
50428bf2672SBrett Creeley /**
50528bf2672SBrett Creeley * ice_init_arfs - initialize aRFS resources
50628bf2672SBrett Creeley * @vsi: the VSI to be forwarded to
50728bf2672SBrett Creeley */
ice_init_arfs(struct ice_vsi * vsi)50828bf2672SBrett Creeley void ice_init_arfs(struct ice_vsi *vsi)
50928bf2672SBrett Creeley {
51028bf2672SBrett Creeley struct hlist_head *arfs_fltr_list;
51128bf2672SBrett Creeley unsigned int i;
51228bf2672SBrett Creeley
51328bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF)
51428bf2672SBrett Creeley return;
51528bf2672SBrett Creeley
51630cba287SLen Baker arfs_fltr_list = kcalloc(ICE_MAX_ARFS_LIST, sizeof(*arfs_fltr_list),
51728bf2672SBrett Creeley GFP_KERNEL);
51828bf2672SBrett Creeley if (!arfs_fltr_list)
51928bf2672SBrett Creeley return;
52028bf2672SBrett Creeley
52128bf2672SBrett Creeley if (ice_init_arfs_cntrs(vsi))
52228bf2672SBrett Creeley goto free_arfs_fltr_list;
52328bf2672SBrett Creeley
52428bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++)
52528bf2672SBrett Creeley INIT_HLIST_HEAD(&arfs_fltr_list[i]);
52628bf2672SBrett Creeley
52728bf2672SBrett Creeley spin_lock_init(&vsi->arfs_lock);
52828bf2672SBrett Creeley
52928bf2672SBrett Creeley vsi->arfs_fltr_list = arfs_fltr_list;
53028bf2672SBrett Creeley
53128bf2672SBrett Creeley return;
53228bf2672SBrett Creeley
53328bf2672SBrett Creeley free_arfs_fltr_list:
53428bf2672SBrett Creeley kfree(arfs_fltr_list);
53528bf2672SBrett Creeley }
53628bf2672SBrett Creeley
53728bf2672SBrett Creeley /**
53828bf2672SBrett Creeley * ice_clear_arfs - clear the aRFS hash table and any memory used for aRFS
53928bf2672SBrett Creeley * @vsi: the VSI to be forwarded to
54028bf2672SBrett Creeley */
ice_clear_arfs(struct ice_vsi * vsi)54128bf2672SBrett Creeley void ice_clear_arfs(struct ice_vsi *vsi)
54228bf2672SBrett Creeley {
54328bf2672SBrett Creeley struct device *dev;
54428bf2672SBrett Creeley unsigned int i;
54528bf2672SBrett Creeley
54628bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF || !vsi->back ||
54728bf2672SBrett Creeley !vsi->arfs_fltr_list)
54828bf2672SBrett Creeley return;
54928bf2672SBrett Creeley
55028bf2672SBrett Creeley dev = ice_pf_to_dev(vsi->back);
55128bf2672SBrett Creeley for (i = 0; i < ICE_MAX_ARFS_LIST; i++) {
55228bf2672SBrett Creeley struct ice_arfs_entry *r;
55328bf2672SBrett Creeley struct hlist_node *n;
55428bf2672SBrett Creeley
55528bf2672SBrett Creeley spin_lock_bh(&vsi->arfs_lock);
55628bf2672SBrett Creeley hlist_for_each_entry_safe(r, n, &vsi->arfs_fltr_list[i],
55728bf2672SBrett Creeley list_entry) {
55828bf2672SBrett Creeley hlist_del(&r->list_entry);
55928bf2672SBrett Creeley devm_kfree(dev, r);
56028bf2672SBrett Creeley }
56128bf2672SBrett Creeley spin_unlock_bh(&vsi->arfs_lock);
56228bf2672SBrett Creeley }
56328bf2672SBrett Creeley
56428bf2672SBrett Creeley kfree(vsi->arfs_fltr_list);
56528bf2672SBrett Creeley vsi->arfs_fltr_list = NULL;
56628bf2672SBrett Creeley kfree(vsi->arfs_last_fltr_id);
56728bf2672SBrett Creeley vsi->arfs_last_fltr_id = NULL;
56828bf2672SBrett Creeley kfree(vsi->arfs_fltr_cntrs);
56928bf2672SBrett Creeley vsi->arfs_fltr_cntrs = NULL;
57028bf2672SBrett Creeley }
57128bf2672SBrett Creeley
57228bf2672SBrett Creeley /**
57328bf2672SBrett Creeley * ice_free_cpu_rx_rmap - free setup CPU reverse map
57428bf2672SBrett Creeley * @vsi: the VSI to be forwarded to
57528bf2672SBrett Creeley */
ice_free_cpu_rx_rmap(struct ice_vsi * vsi)57628bf2672SBrett Creeley void ice_free_cpu_rx_rmap(struct ice_vsi *vsi)
57728bf2672SBrett Creeley {
57828bf2672SBrett Creeley struct net_device *netdev;
57928bf2672SBrett Creeley
580d7442f51SAlexander Lobakin if (!vsi || vsi->type != ICE_VSI_PF)
58128bf2672SBrett Creeley return;
58228bf2672SBrett Creeley
58328bf2672SBrett Creeley netdev = vsi->netdev;
5841e23f076SAnirudh Venkataramanan if (!netdev || !netdev->rx_cpu_rmap)
58528bf2672SBrett Creeley return;
58628bf2672SBrett Creeley
58728bf2672SBrett Creeley free_irq_cpu_rmap(netdev->rx_cpu_rmap);
58828bf2672SBrett Creeley netdev->rx_cpu_rmap = NULL;
58928bf2672SBrett Creeley }
59028bf2672SBrett Creeley
59128bf2672SBrett Creeley /**
59228bf2672SBrett Creeley * ice_set_cpu_rx_rmap - setup CPU reverse map for each queue
59328bf2672SBrett Creeley * @vsi: the VSI to be forwarded to
59428bf2672SBrett Creeley */
ice_set_cpu_rx_rmap(struct ice_vsi * vsi)59528bf2672SBrett Creeley int ice_set_cpu_rx_rmap(struct ice_vsi *vsi)
59628bf2672SBrett Creeley {
59728bf2672SBrett Creeley struct net_device *netdev;
59828bf2672SBrett Creeley struct ice_pf *pf;
599*4aad5335SPiotr Raczynski int i;
60028bf2672SBrett Creeley
60128bf2672SBrett Creeley if (!vsi || vsi->type != ICE_VSI_PF)
602d7442f51SAlexander Lobakin return 0;
60328bf2672SBrett Creeley
60428bf2672SBrett Creeley pf = vsi->back;
60528bf2672SBrett Creeley netdev = vsi->netdev;
6061e23f076SAnirudh Venkataramanan if (!pf || !netdev || !vsi->num_q_vectors)
60728bf2672SBrett Creeley return -EINVAL;
60828bf2672SBrett Creeley
60928bf2672SBrett Creeley netdev_dbg(netdev, "Setup CPU RMAP: vsi type 0x%x, ifname %s, q_vectors %d\n",
61028bf2672SBrett Creeley vsi->type, netdev->name, vsi->num_q_vectors);
61128bf2672SBrett Creeley
61228bf2672SBrett Creeley netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(vsi->num_q_vectors);
61328bf2672SBrett Creeley if (unlikely(!netdev->rx_cpu_rmap))
61428bf2672SBrett Creeley return -EINVAL;
61528bf2672SBrett Creeley
6162faf63b6SMaciej Fijalkowski ice_for_each_q_vector(vsi, i)
61728bf2672SBrett Creeley if (irq_cpu_rmap_add(netdev->rx_cpu_rmap,
618*4aad5335SPiotr Raczynski vsi->q_vectors[i]->irq.virq)) {
61928bf2672SBrett Creeley ice_free_cpu_rx_rmap(vsi);
62028bf2672SBrett Creeley return -EINVAL;
62128bf2672SBrett Creeley }
62228bf2672SBrett Creeley
62328bf2672SBrett Creeley return 0;
62428bf2672SBrett Creeley }
62528bf2672SBrett Creeley
62628bf2672SBrett Creeley /**
62728bf2672SBrett Creeley * ice_remove_arfs - remove/clear all aRFS resources
62828bf2672SBrett Creeley * @pf: device private structure
62928bf2672SBrett Creeley */
ice_remove_arfs(struct ice_pf * pf)63028bf2672SBrett Creeley void ice_remove_arfs(struct ice_pf *pf)
63128bf2672SBrett Creeley {
63228bf2672SBrett Creeley struct ice_vsi *pf_vsi;
63328bf2672SBrett Creeley
63428bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf);
63528bf2672SBrett Creeley if (!pf_vsi)
63628bf2672SBrett Creeley return;
63728bf2672SBrett Creeley
63828bf2672SBrett Creeley ice_clear_arfs(pf_vsi);
63928bf2672SBrett Creeley }
64028bf2672SBrett Creeley
64128bf2672SBrett Creeley /**
64228bf2672SBrett Creeley * ice_rebuild_arfs - remove/clear all aRFS resources and rebuild after reset
64328bf2672SBrett Creeley * @pf: device private structure
64428bf2672SBrett Creeley */
ice_rebuild_arfs(struct ice_pf * pf)64528bf2672SBrett Creeley void ice_rebuild_arfs(struct ice_pf *pf)
64628bf2672SBrett Creeley {
64728bf2672SBrett Creeley struct ice_vsi *pf_vsi;
64828bf2672SBrett Creeley
64928bf2672SBrett Creeley pf_vsi = ice_get_main_vsi(pf);
65028bf2672SBrett Creeley if (!pf_vsi)
65128bf2672SBrett Creeley return;
65228bf2672SBrett Creeley
65328bf2672SBrett Creeley ice_remove_arfs(pf);
65428bf2672SBrett Creeley ice_init_arfs(pf_vsi);
65528bf2672SBrett Creeley }
656