1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 263118527SGovindarajulu Varadarajan #include <linux/if.h> 363118527SGovindarajulu Varadarajan #include <linux/if_ether.h> 463118527SGovindarajulu Varadarajan #include <linux/if_link.h> 563118527SGovindarajulu Varadarajan #include <linux/netdevice.h> 663118527SGovindarajulu Varadarajan #include <linux/in.h> 763118527SGovindarajulu Varadarajan #include <linux/types.h> 863118527SGovindarajulu Varadarajan #include <linux/skbuff.h> 91bd758ebSJiri Pirko #include <net/flow_dissector.h> 1063118527SGovindarajulu Varadarajan #include "enic_res.h" 1163118527SGovindarajulu Varadarajan #include "enic_clsf.h" 1263118527SGovindarajulu Varadarajan 1363118527SGovindarajulu Varadarajan /* enic_addfltr_5t - Add ipv4 5tuple filter 1463118527SGovindarajulu Varadarajan * @enic: enic struct of vnic 1563118527SGovindarajulu Varadarajan * @keys: flow_keys of ipv4 5tuple 1663118527SGovindarajulu Varadarajan * @rq: rq number to steer to 1763118527SGovindarajulu Varadarajan * 1863118527SGovindarajulu Varadarajan * This function returns filter_id(hardware_id) of the filter 1929044b58SGeert Uytterhoeven * added. In case of error it returns a negative number. 2063118527SGovindarajulu Varadarajan */ 2163118527SGovindarajulu Varadarajan int enic_addfltr_5t(struct enic *enic, struct flow_keys *keys, u16 rq) 2263118527SGovindarajulu Varadarajan { 2363118527SGovindarajulu Varadarajan int res; 2463118527SGovindarajulu Varadarajan struct filter data; 2563118527SGovindarajulu Varadarajan 2606635a35SJiri Pirko switch (keys->basic.ip_proto) { 2763118527SGovindarajulu Varadarajan case IPPROTO_TCP: 2863118527SGovindarajulu Varadarajan data.u.ipv4.protocol = PROTO_TCP; 2963118527SGovindarajulu Varadarajan break; 3063118527SGovindarajulu Varadarajan case IPPROTO_UDP: 3163118527SGovindarajulu Varadarajan data.u.ipv4.protocol = PROTO_UDP; 3263118527SGovindarajulu Varadarajan break; 3363118527SGovindarajulu Varadarajan default: 3463118527SGovindarajulu Varadarajan return -EPROTONOSUPPORT; 3563118527SGovindarajulu Varadarajan }; 3663118527SGovindarajulu Varadarajan data.type = FILTER_IPV4_5TUPLE; 37c3f83241STom Herbert data.u.ipv4.src_addr = ntohl(keys->addrs.v4addrs.src); 38c3f83241STom Herbert data.u.ipv4.dst_addr = ntohl(keys->addrs.v4addrs.dst); 3959346afeSJiri Pirko data.u.ipv4.src_port = ntohs(keys->ports.src); 4059346afeSJiri Pirko data.u.ipv4.dst_port = ntohs(keys->ports.dst); 4163118527SGovindarajulu Varadarajan data.u.ipv4.flags = FILTER_FIELDS_IPV4_5TUPLE; 4263118527SGovindarajulu Varadarajan 4363118527SGovindarajulu Varadarajan spin_lock_bh(&enic->devcmd_lock); 4463118527SGovindarajulu Varadarajan res = vnic_dev_classifier(enic->vdev, CLSF_ADD, &rq, &data); 4563118527SGovindarajulu Varadarajan spin_unlock_bh(&enic->devcmd_lock); 4663118527SGovindarajulu Varadarajan res = (res == 0) ? rq : res; 4763118527SGovindarajulu Varadarajan 4863118527SGovindarajulu Varadarajan return res; 4963118527SGovindarajulu Varadarajan } 5063118527SGovindarajulu Varadarajan 5163118527SGovindarajulu Varadarajan /* enic_delfltr - Delete clsf filter 5263118527SGovindarajulu Varadarajan * @enic: enic struct of vnic 5363118527SGovindarajulu Varadarajan * @filter_id: filter_is(hardware_id) of filter to be deleted 5463118527SGovindarajulu Varadarajan * 5563118527SGovindarajulu Varadarajan * This function returns zero in case of success, negative number incase of 5663118527SGovindarajulu Varadarajan * error. 5763118527SGovindarajulu Varadarajan */ 5863118527SGovindarajulu Varadarajan int enic_delfltr(struct enic *enic, u16 filter_id) 5963118527SGovindarajulu Varadarajan { 6063118527SGovindarajulu Varadarajan int ret; 6163118527SGovindarajulu Varadarajan 6263118527SGovindarajulu Varadarajan spin_lock_bh(&enic->devcmd_lock); 6363118527SGovindarajulu Varadarajan ret = vnic_dev_classifier(enic->vdev, CLSF_DEL, &filter_id, NULL); 6463118527SGovindarajulu Varadarajan spin_unlock_bh(&enic->devcmd_lock); 6563118527SGovindarajulu Varadarajan 6663118527SGovindarajulu Varadarajan return ret; 6763118527SGovindarajulu Varadarajan } 68a145df23SGovindarajulu Varadarajan 69e3e5af33SGovindarajulu Varadarajan /* enic_rfs_flw_tbl_init - initialize enic->rfs_h members 70e3e5af33SGovindarajulu Varadarajan * @enic: enic data 71e3e5af33SGovindarajulu Varadarajan */ 72e3e5af33SGovindarajulu Varadarajan void enic_rfs_flw_tbl_init(struct enic *enic) 73e3e5af33SGovindarajulu Varadarajan { 74e3e5af33SGovindarajulu Varadarajan int i; 75e3e5af33SGovindarajulu Varadarajan 76e3e5af33SGovindarajulu Varadarajan spin_lock_init(&enic->rfs_h.lock); 77e3e5af33SGovindarajulu Varadarajan for (i = 0; i <= ENIC_RFS_FLW_MASK; i++) 78e3e5af33SGovindarajulu Varadarajan INIT_HLIST_HEAD(&enic->rfs_h.ht_head[i]); 79e3e5af33SGovindarajulu Varadarajan enic->rfs_h.max = enic->config.num_arfs; 80e3e5af33SGovindarajulu Varadarajan enic->rfs_h.free = enic->rfs_h.max; 81e3e5af33SGovindarajulu Varadarajan enic->rfs_h.toclean = 0; 82e3e5af33SGovindarajulu Varadarajan } 83e3e5af33SGovindarajulu Varadarajan 84e3e5af33SGovindarajulu Varadarajan void enic_rfs_flw_tbl_free(struct enic *enic) 85e3e5af33SGovindarajulu Varadarajan { 86e3e5af33SGovindarajulu Varadarajan int i; 87e3e5af33SGovindarajulu Varadarajan 88e3e5af33SGovindarajulu Varadarajan enic_rfs_timer_stop(enic); 89b6931c9bSGovindarajulu Varadarajan spin_lock_bh(&enic->rfs_h.lock); 90e3e5af33SGovindarajulu Varadarajan for (i = 0; i < (1 << ENIC_RFS_FLW_BITSHIFT); i++) { 91e3e5af33SGovindarajulu Varadarajan struct hlist_head *hhead; 92e3e5af33SGovindarajulu Varadarajan struct hlist_node *tmp; 93e3e5af33SGovindarajulu Varadarajan struct enic_rfs_fltr_node *n; 94e3e5af33SGovindarajulu Varadarajan 95e3e5af33SGovindarajulu Varadarajan hhead = &enic->rfs_h.ht_head[i]; 96e3e5af33SGovindarajulu Varadarajan hlist_for_each_entry_safe(n, tmp, hhead, node) { 97e3e5af33SGovindarajulu Varadarajan enic_delfltr(enic, n->fltr_id); 98e3e5af33SGovindarajulu Varadarajan hlist_del(&n->node); 99e3e5af33SGovindarajulu Varadarajan kfree(n); 1003256d29fSGovindarajulu Varadarajan enic->rfs_h.free++; 101e3e5af33SGovindarajulu Varadarajan } 102e3e5af33SGovindarajulu Varadarajan } 103b6931c9bSGovindarajulu Varadarajan spin_unlock_bh(&enic->rfs_h.lock); 104e3e5af33SGovindarajulu Varadarajan } 105e3e5af33SGovindarajulu Varadarajan 1063762ff8fSGovindarajulu Varadarajan struct enic_rfs_fltr_node *htbl_fltr_search(struct enic *enic, u16 fltr_id) 1073762ff8fSGovindarajulu Varadarajan { 1083762ff8fSGovindarajulu Varadarajan int i; 1093762ff8fSGovindarajulu Varadarajan 1103762ff8fSGovindarajulu Varadarajan for (i = 0; i < (1 << ENIC_RFS_FLW_BITSHIFT); i++) { 1113762ff8fSGovindarajulu Varadarajan struct hlist_head *hhead; 1123762ff8fSGovindarajulu Varadarajan struct hlist_node *tmp; 1133762ff8fSGovindarajulu Varadarajan struct enic_rfs_fltr_node *n; 1143762ff8fSGovindarajulu Varadarajan 1153762ff8fSGovindarajulu Varadarajan hhead = &enic->rfs_h.ht_head[i]; 1163762ff8fSGovindarajulu Varadarajan hlist_for_each_entry_safe(n, tmp, hhead, node) 1173762ff8fSGovindarajulu Varadarajan if (n->fltr_id == fltr_id) 1183762ff8fSGovindarajulu Varadarajan return n; 1193762ff8fSGovindarajulu Varadarajan } 1203762ff8fSGovindarajulu Varadarajan 1213762ff8fSGovindarajulu Varadarajan return NULL; 1223762ff8fSGovindarajulu Varadarajan } 1233762ff8fSGovindarajulu Varadarajan 124a145df23SGovindarajulu Varadarajan #ifdef CONFIG_RFS_ACCEL 12586cb30ecSKees Cook void enic_flow_may_expire(struct timer_list *t) 126a145df23SGovindarajulu Varadarajan { 12786cb30ecSKees Cook struct enic *enic = from_timer(enic, t, rfs_h.rfs_may_expire); 128a145df23SGovindarajulu Varadarajan bool res; 129a145df23SGovindarajulu Varadarajan int j; 130a145df23SGovindarajulu Varadarajan 131b6931c9bSGovindarajulu Varadarajan spin_lock_bh(&enic->rfs_h.lock); 132a145df23SGovindarajulu Varadarajan for (j = 0; j < ENIC_CLSF_EXPIRE_COUNT; j++) { 133a145df23SGovindarajulu Varadarajan struct hlist_head *hhead; 134a145df23SGovindarajulu Varadarajan struct hlist_node *tmp; 135a145df23SGovindarajulu Varadarajan struct enic_rfs_fltr_node *n; 136a145df23SGovindarajulu Varadarajan 137a145df23SGovindarajulu Varadarajan hhead = &enic->rfs_h.ht_head[enic->rfs_h.toclean++]; 138a145df23SGovindarajulu Varadarajan hlist_for_each_entry_safe(n, tmp, hhead, node) { 139a145df23SGovindarajulu Varadarajan res = rps_may_expire_flow(enic->netdev, n->rq_id, 140a145df23SGovindarajulu Varadarajan n->flow_id, n->fltr_id); 141a145df23SGovindarajulu Varadarajan if (res) { 142a145df23SGovindarajulu Varadarajan res = enic_delfltr(enic, n->fltr_id); 143a145df23SGovindarajulu Varadarajan if (unlikely(res)) 144a145df23SGovindarajulu Varadarajan continue; 145a145df23SGovindarajulu Varadarajan hlist_del(&n->node); 146a145df23SGovindarajulu Varadarajan kfree(n); 147a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 148a145df23SGovindarajulu Varadarajan } 149a145df23SGovindarajulu Varadarajan } 150a145df23SGovindarajulu Varadarajan } 151b6931c9bSGovindarajulu Varadarajan spin_unlock_bh(&enic->rfs_h.lock); 152a145df23SGovindarajulu Varadarajan mod_timer(&enic->rfs_h.rfs_may_expire, jiffies + HZ/4); 153a145df23SGovindarajulu Varadarajan } 154a145df23SGovindarajulu Varadarajan 155a145df23SGovindarajulu Varadarajan static struct enic_rfs_fltr_node *htbl_key_search(struct hlist_head *h, 156a145df23SGovindarajulu Varadarajan struct flow_keys *k) 157a145df23SGovindarajulu Varadarajan { 158a145df23SGovindarajulu Varadarajan struct enic_rfs_fltr_node *tpos; 159a145df23SGovindarajulu Varadarajan 160a145df23SGovindarajulu Varadarajan hlist_for_each_entry(tpos, h, node) 161c3f83241STom Herbert if (tpos->keys.addrs.v4addrs.src == k->addrs.v4addrs.src && 162c3f83241STom Herbert tpos->keys.addrs.v4addrs.dst == k->addrs.v4addrs.dst && 16306635a35SJiri Pirko tpos->keys.ports.ports == k->ports.ports && 16406635a35SJiri Pirko tpos->keys.basic.ip_proto == k->basic.ip_proto && 16506635a35SJiri Pirko tpos->keys.basic.n_proto == k->basic.n_proto) 166a145df23SGovindarajulu Varadarajan return tpos; 167a145df23SGovindarajulu Varadarajan return NULL; 168a145df23SGovindarajulu Varadarajan } 169a145df23SGovindarajulu Varadarajan 170a145df23SGovindarajulu Varadarajan int enic_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, 171a145df23SGovindarajulu Varadarajan u16 rxq_index, u32 flow_id) 172a145df23SGovindarajulu Varadarajan { 173a145df23SGovindarajulu Varadarajan struct flow_keys keys; 174a145df23SGovindarajulu Varadarajan struct enic_rfs_fltr_node *n; 175a145df23SGovindarajulu Varadarajan struct enic *enic; 176a145df23SGovindarajulu Varadarajan u16 tbl_idx; 177a145df23SGovindarajulu Varadarajan int res, i; 178a145df23SGovindarajulu Varadarajan 179a145df23SGovindarajulu Varadarajan enic = netdev_priv(dev); 180cd79a238STom Herbert res = skb_flow_dissect_flow_keys(skb, &keys, 0); 18106635a35SJiri Pirko if (!res || keys.basic.n_proto != htons(ETH_P_IP) || 18206635a35SJiri Pirko (keys.basic.ip_proto != IPPROTO_TCP && 18306635a35SJiri Pirko keys.basic.ip_proto != IPPROTO_UDP)) 184a145df23SGovindarajulu Varadarajan return -EPROTONOSUPPORT; 185a145df23SGovindarajulu Varadarajan 186a145df23SGovindarajulu Varadarajan tbl_idx = skb_get_hash_raw(skb) & ENIC_RFS_FLW_MASK; 187b6931c9bSGovindarajulu Varadarajan spin_lock_bh(&enic->rfs_h.lock); 188a145df23SGovindarajulu Varadarajan n = htbl_key_search(&enic->rfs_h.ht_head[tbl_idx], &keys); 189a145df23SGovindarajulu Varadarajan 190a145df23SGovindarajulu Varadarajan if (n) { /* entry already present */ 191a145df23SGovindarajulu Varadarajan if (rxq_index == n->rq_id) { 192a145df23SGovindarajulu Varadarajan res = -EEXIST; 193a145df23SGovindarajulu Varadarajan goto ret_unlock; 194a145df23SGovindarajulu Varadarajan } 195a145df23SGovindarajulu Varadarajan 196a145df23SGovindarajulu Varadarajan /* desired rq changed for the flow, we need to delete 197a145df23SGovindarajulu Varadarajan * old fltr and add new one 198a145df23SGovindarajulu Varadarajan * 199a145df23SGovindarajulu Varadarajan * The moment we delete the fltr, the upcoming pkts 200a145df23SGovindarajulu Varadarajan * are put it default rq based on rss. When we add 201a145df23SGovindarajulu Varadarajan * new filter, upcoming pkts are put in desired queue. 202a145df23SGovindarajulu Varadarajan * This could cause ooo pkts. 203a145df23SGovindarajulu Varadarajan * 204a145df23SGovindarajulu Varadarajan * Lets 1st try adding new fltr and then del old one. 205a145df23SGovindarajulu Varadarajan */ 206a145df23SGovindarajulu Varadarajan i = --enic->rfs_h.free; 207a145df23SGovindarajulu Varadarajan /* clsf tbl is full, we have to del old fltr first*/ 208a145df23SGovindarajulu Varadarajan if (unlikely(i < 0)) { 209a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 210a145df23SGovindarajulu Varadarajan res = enic_delfltr(enic, n->fltr_id); 211a145df23SGovindarajulu Varadarajan if (unlikely(res < 0)) 212a145df23SGovindarajulu Varadarajan goto ret_unlock; 213a145df23SGovindarajulu Varadarajan res = enic_addfltr_5t(enic, &keys, rxq_index); 214a145df23SGovindarajulu Varadarajan if (res < 0) { 215a145df23SGovindarajulu Varadarajan hlist_del(&n->node); 216a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 217a145df23SGovindarajulu Varadarajan goto ret_unlock; 218a145df23SGovindarajulu Varadarajan } 219a145df23SGovindarajulu Varadarajan /* add new fltr 1st then del old fltr */ 220a145df23SGovindarajulu Varadarajan } else { 221a145df23SGovindarajulu Varadarajan int ret; 222a145df23SGovindarajulu Varadarajan 223a145df23SGovindarajulu Varadarajan res = enic_addfltr_5t(enic, &keys, rxq_index); 224a145df23SGovindarajulu Varadarajan if (res < 0) { 225a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 226a145df23SGovindarajulu Varadarajan goto ret_unlock; 227a145df23SGovindarajulu Varadarajan } 228a145df23SGovindarajulu Varadarajan ret = enic_delfltr(enic, n->fltr_id); 229a145df23SGovindarajulu Varadarajan /* deleting old fltr failed. Add old fltr to list. 230a145df23SGovindarajulu Varadarajan * enic_flow_may_expire() will try to delete it later. 231a145df23SGovindarajulu Varadarajan */ 232a145df23SGovindarajulu Varadarajan if (unlikely(ret < 0)) { 233a145df23SGovindarajulu Varadarajan struct enic_rfs_fltr_node *d; 234a145df23SGovindarajulu Varadarajan struct hlist_head *head; 235a145df23SGovindarajulu Varadarajan 236a145df23SGovindarajulu Varadarajan head = &enic->rfs_h.ht_head[tbl_idx]; 237a145df23SGovindarajulu Varadarajan d = kmalloc(sizeof(*d), GFP_ATOMIC); 238a145df23SGovindarajulu Varadarajan if (d) { 239a145df23SGovindarajulu Varadarajan d->fltr_id = n->fltr_id; 240a145df23SGovindarajulu Varadarajan INIT_HLIST_NODE(&d->node); 241a145df23SGovindarajulu Varadarajan hlist_add_head(&d->node, head); 242a145df23SGovindarajulu Varadarajan } 243a145df23SGovindarajulu Varadarajan } else { 244a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 245a145df23SGovindarajulu Varadarajan } 246a145df23SGovindarajulu Varadarajan } 247a145df23SGovindarajulu Varadarajan n->rq_id = rxq_index; 248a145df23SGovindarajulu Varadarajan n->fltr_id = res; 249a145df23SGovindarajulu Varadarajan n->flow_id = flow_id; 250a145df23SGovindarajulu Varadarajan /* entry not present */ 251a145df23SGovindarajulu Varadarajan } else { 252a145df23SGovindarajulu Varadarajan i = --enic->rfs_h.free; 253a145df23SGovindarajulu Varadarajan if (i <= 0) { 254a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 255a145df23SGovindarajulu Varadarajan res = -EBUSY; 256a145df23SGovindarajulu Varadarajan goto ret_unlock; 257a145df23SGovindarajulu Varadarajan } 258a145df23SGovindarajulu Varadarajan 259a145df23SGovindarajulu Varadarajan n = kmalloc(sizeof(*n), GFP_ATOMIC); 260a145df23SGovindarajulu Varadarajan if (!n) { 261a145df23SGovindarajulu Varadarajan res = -ENOMEM; 262a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 263a145df23SGovindarajulu Varadarajan goto ret_unlock; 264a145df23SGovindarajulu Varadarajan } 265a145df23SGovindarajulu Varadarajan 266a145df23SGovindarajulu Varadarajan res = enic_addfltr_5t(enic, &keys, rxq_index); 267a145df23SGovindarajulu Varadarajan if (res < 0) { 268a145df23SGovindarajulu Varadarajan kfree(n); 269a145df23SGovindarajulu Varadarajan enic->rfs_h.free++; 270a145df23SGovindarajulu Varadarajan goto ret_unlock; 271a145df23SGovindarajulu Varadarajan } 272a145df23SGovindarajulu Varadarajan n->rq_id = rxq_index; 273a145df23SGovindarajulu Varadarajan n->fltr_id = res; 274a145df23SGovindarajulu Varadarajan n->flow_id = flow_id; 275a145df23SGovindarajulu Varadarajan n->keys = keys; 276a145df23SGovindarajulu Varadarajan INIT_HLIST_NODE(&n->node); 277a145df23SGovindarajulu Varadarajan hlist_add_head(&n->node, &enic->rfs_h.ht_head[tbl_idx]); 278a145df23SGovindarajulu Varadarajan } 279a145df23SGovindarajulu Varadarajan 280a145df23SGovindarajulu Varadarajan ret_unlock: 281b6931c9bSGovindarajulu Varadarajan spin_unlock_bh(&enic->rfs_h.lock); 282a145df23SGovindarajulu Varadarajan return res; 283a145df23SGovindarajulu Varadarajan } 284a145df23SGovindarajulu Varadarajan 285a145df23SGovindarajulu Varadarajan #endif /* CONFIG_RFS_ACCEL */ 286