1c9422999SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2e6445719SPravin B Shelar /* 39b996e54SPravin B Shelar * Copyright (c) 2007-2014 Nicira, Inc. 4e6445719SPravin B Shelar */ 5e6445719SPravin B Shelar 6e6445719SPravin B Shelar #include "flow.h" 7e6445719SPravin B Shelar #include "datapath.h" 834ae932aSThomas Graf #include "flow_netlink.h" 9e6445719SPravin B Shelar #include <linux/uaccess.h> 10e6445719SPravin B Shelar #include <linux/netdevice.h> 11e6445719SPravin B Shelar #include <linux/etherdevice.h> 12e6445719SPravin B Shelar #include <linux/if_ether.h> 13e6445719SPravin B Shelar #include <linux/if_vlan.h> 14e6445719SPravin B Shelar #include <net/llc_pdu.h> 15e6445719SPravin B Shelar #include <linux/kernel.h> 1687545899SDaniel Borkmann #include <linux/jhash.h> 17e6445719SPravin B Shelar #include <linux/jiffies.h> 18e6445719SPravin B Shelar #include <linux/llc.h> 19e6445719SPravin B Shelar #include <linux/module.h> 20e6445719SPravin B Shelar #include <linux/in.h> 21e6445719SPravin B Shelar #include <linux/rcupdate.h> 22db74a333SThadeu Lima de Souza Cascardo #include <linux/cpumask.h> 23e6445719SPravin B Shelar #include <linux/if_arp.h> 24e6445719SPravin B Shelar #include <linux/ip.h> 25e6445719SPravin B Shelar #include <linux/ipv6.h> 26e6445719SPravin B Shelar #include <linux/sctp.h> 27e6445719SPravin B Shelar #include <linux/tcp.h> 28e6445719SPravin B Shelar #include <linux/udp.h> 29e6445719SPravin B Shelar #include <linux/icmp.h> 30e6445719SPravin B Shelar #include <linux/icmpv6.h> 31e6445719SPravin B Shelar #include <linux/rculist.h> 32eac87c41SEelco Chaudron #include <linux/sort.h> 33e6445719SPravin B Shelar #include <net/ip.h> 34e6445719SPravin B Shelar #include <net/ipv6.h> 35e6445719SPravin B Shelar #include <net/ndisc.h> 36e6445719SPravin B Shelar 37b637e498SPravin B Shelar #define TBL_MIN_BUCKETS 1024 384bc63b1bSTonghao Zhang #define MASK_ARRAY_SIZE_MIN 16 39b637e498SPravin B Shelar #define REHASH_INTERVAL (10 * 60 * HZ) 40b637e498SPravin B Shelar 4104b7d136STonghao Zhang #define MC_HASH_SHIFT 8 4204b7d136STonghao Zhang #define MC_HASH_ENTRIES (1u << MC_HASH_SHIFT) 4304b7d136STonghao Zhang #define MC_HASH_SEGS ((sizeof(uint32_t) * 8) / MC_HASH_SHIFT) 4404b7d136STonghao Zhang 45e6445719SPravin B Shelar static struct kmem_cache *flow_cache; 4663e7959cSJarno Rajahalme struct kmem_cache *flow_stats_cache __read_mostly; 47e6445719SPravin B Shelar 48e6445719SPravin B Shelar static u16 range_n_bytes(const struct sw_flow_key_range *range) 49e6445719SPravin B Shelar { 50e6445719SPravin B Shelar return range->end - range->start; 51e6445719SPravin B Shelar } 52e6445719SPravin B Shelar 53e6445719SPravin B Shelar void ovs_flow_mask_key(struct sw_flow_key *dst, const struct sw_flow_key *src, 54ae5f2fb1SJesse Gross bool full, const struct sw_flow_mask *mask) 55e6445719SPravin B Shelar { 56ae5f2fb1SJesse Gross int start = full ? 0 : mask->range.start; 57ae5f2fb1SJesse Gross int len = full ? sizeof *dst : range_n_bytes(&mask->range); 58ae5f2fb1SJesse Gross const long *m = (const long *)((const u8 *)&mask->key + start); 59ae5f2fb1SJesse Gross const long *s = (const long *)((const u8 *)src + start); 60ae5f2fb1SJesse Gross long *d = (long *)((u8 *)dst + start); 61e6445719SPravin B Shelar int i; 62e6445719SPravin B Shelar 63ae5f2fb1SJesse Gross /* If 'full' is true then all of 'dst' is fully initialized. Otherwise, 64ae5f2fb1SJesse Gross * if 'full' is false the memory outside of the 'mask->range' is left 65ae5f2fb1SJesse Gross * uninitialized. This can be used as an optimization when further 66ae5f2fb1SJesse Gross * operations on 'dst' only use contents within 'mask->range'. 67e6445719SPravin B Shelar */ 68ae5f2fb1SJesse Gross for (i = 0; i < len; i += sizeof(long)) 69e6445719SPravin B Shelar *d++ = *s++ & *m++; 70e6445719SPravin B Shelar } 71e6445719SPravin B Shelar 7223dabf88SJarno Rajahalme struct sw_flow *ovs_flow_alloc(void) 73e6445719SPravin B Shelar { 74e6445719SPravin B Shelar struct sw_flow *flow; 75aef833c5SPablo Neira Ayuso struct sw_flow_stats *stats; 76e6445719SPravin B Shelar 77db74a333SThadeu Lima de Souza Cascardo flow = kmem_cache_zalloc(flow_cache, GFP_KERNEL); 78e6445719SPravin B Shelar if (!flow) 79e6445719SPravin B Shelar return ERR_PTR(-ENOMEM); 80e6445719SPravin B Shelar 81db74a333SThadeu Lima de Souza Cascardo flow->stats_last_writer = -1; 82e6445719SPravin B Shelar 8363e7959cSJarno Rajahalme /* Initialize the default stat node. */ 8463e7959cSJarno Rajahalme stats = kmem_cache_alloc_node(flow_stats_cache, 85598c12d0SKonstantin Khlebnikov GFP_KERNEL | __GFP_ZERO, 86598c12d0SKonstantin Khlebnikov node_online(0) ? 0 : NUMA_NO_NODE); 8763e7959cSJarno Rajahalme if (!stats) 88e298e505SPravin B Shelar goto err; 89e298e505SPravin B Shelar 9063e7959cSJarno Rajahalme spin_lock_init(&stats->lock); 91e298e505SPravin B Shelar 9263e7959cSJarno Rajahalme RCU_INIT_POINTER(flow->stats[0], stats); 9363e7959cSJarno Rajahalme 94c4b2bf6bSTonghao Zhang cpumask_set_cpu(0, &flow->cpu_used_mask); 95c4b2bf6bSTonghao Zhang 96e6445719SPravin B Shelar return flow; 97e298e505SPravin B Shelar err: 98ece37c87SWei Yongjun kmem_cache_free(flow_cache, flow); 99e298e505SPravin B Shelar return ERR_PTR(-ENOMEM); 100e6445719SPravin B Shelar } 101e6445719SPravin B Shelar 10212eb18f7SThomas Graf int ovs_flow_tbl_count(const struct flow_table *table) 103b637e498SPravin B Shelar { 104b637e498SPravin B Shelar return table->count; 105b637e498SPravin B Shelar } 106b637e498SPravin B Shelar 107e6445719SPravin B Shelar static void flow_free(struct sw_flow *flow) 108e6445719SPravin B Shelar { 109db74a333SThadeu Lima de Souza Cascardo int cpu; 11063e7959cSJarno Rajahalme 11174ed7ab9SJoe Stringer if (ovs_identifier_is_key(&flow->id)) 11274ed7ab9SJoe Stringer kfree(flow->id.unmasked_key); 11334ae932aSThomas Graf if (flow->sf_acts) 11434ae932aSThomas Graf ovs_nla_free_flow_actions((struct sw_flow_actions __force *)flow->sf_acts); 115db74a333SThadeu Lima de Souza Cascardo /* We open code this to make sure cpu 0 is always considered */ 116c4b2bf6bSTonghao Zhang for (cpu = 0; cpu < nr_cpu_ids; cpu = cpumask_next(cpu, &flow->cpu_used_mask)) 117db74a333SThadeu Lima de Souza Cascardo if (flow->stats[cpu]) 11863e7959cSJarno Rajahalme kmem_cache_free(flow_stats_cache, 119aef833c5SPablo Neira Ayuso (struct sw_flow_stats __force *)flow->stats[cpu]); 120e6445719SPravin B Shelar kmem_cache_free(flow_cache, flow); 121e6445719SPravin B Shelar } 122e6445719SPravin B Shelar 123e6445719SPravin B Shelar static void rcu_free_flow_callback(struct rcu_head *rcu) 124e6445719SPravin B Shelar { 125e6445719SPravin B Shelar struct sw_flow *flow = container_of(rcu, struct sw_flow, rcu); 126e6445719SPravin B Shelar 127e6445719SPravin B Shelar flow_free(flow); 128e6445719SPravin B Shelar } 129e6445719SPravin B Shelar 130e80857ccSAndy Zhou void ovs_flow_free(struct sw_flow *flow, bool deferred) 131618ed0c8SPravin B Shelar { 132e80857ccSAndy Zhou if (!flow) 133618ed0c8SPravin B Shelar return; 134618ed0c8SPravin B Shelar 135e6445719SPravin B Shelar if (deferred) 136e6445719SPravin B Shelar call_rcu(&flow->rcu, rcu_free_flow_callback); 137e6445719SPravin B Shelar else 138e6445719SPravin B Shelar flow_free(flow); 139e6445719SPravin B Shelar } 140e6445719SPravin B Shelar 141b637e498SPravin B Shelar static void __table_instance_destroy(struct table_instance *ti) 142e6445719SPravin B Shelar { 143ee9c5e67SKent Overstreet kvfree(ti->buckets); 144b637e498SPravin B Shelar kfree(ti); 145e6445719SPravin B Shelar } 146e6445719SPravin B Shelar 147b637e498SPravin B Shelar static struct table_instance *table_instance_alloc(int new_size) 148e6445719SPravin B Shelar { 149b637e498SPravin B Shelar struct table_instance *ti = kmalloc(sizeof(*ti), GFP_KERNEL); 150ee9c5e67SKent Overstreet int i; 151e6445719SPravin B Shelar 152b637e498SPravin B Shelar if (!ti) 153e6445719SPravin B Shelar return NULL; 154e6445719SPravin B Shelar 155ee9c5e67SKent Overstreet ti->buckets = kvmalloc_array(new_size, sizeof(struct hlist_head), 156ee9c5e67SKent Overstreet GFP_KERNEL); 157b637e498SPravin B Shelar if (!ti->buckets) { 158b637e498SPravin B Shelar kfree(ti); 159e6445719SPravin B Shelar return NULL; 160e6445719SPravin B Shelar } 161ee9c5e67SKent Overstreet 162ee9c5e67SKent Overstreet for (i = 0; i < new_size; i++) 163ee9c5e67SKent Overstreet INIT_HLIST_HEAD(&ti->buckets[i]); 164ee9c5e67SKent Overstreet 165b637e498SPravin B Shelar ti->n_buckets = new_size; 166b637e498SPravin B Shelar ti->node_ver = 0; 167b637e498SPravin B Shelar ti->keep_flows = false; 168b637e498SPravin B Shelar get_random_bytes(&ti->hash_seed, sizeof(u32)); 169b637e498SPravin B Shelar 170b637e498SPravin B Shelar return ti; 171b637e498SPravin B Shelar } 172b637e498SPravin B Shelar 173eac87c41SEelco Chaudron static void __mask_array_destroy(struct mask_array *ma) 174eac87c41SEelco Chaudron { 175eac87c41SEelco Chaudron free_percpu(ma->masks_usage_cntr); 176eac87c41SEelco Chaudron kfree(ma); 177eac87c41SEelco Chaudron } 178eac87c41SEelco Chaudron 179eac87c41SEelco Chaudron static void mask_array_rcu_cb(struct rcu_head *rcu) 180eac87c41SEelco Chaudron { 181eac87c41SEelco Chaudron struct mask_array *ma = container_of(rcu, struct mask_array, rcu); 182eac87c41SEelco Chaudron 183eac87c41SEelco Chaudron __mask_array_destroy(ma); 184eac87c41SEelco Chaudron } 185eac87c41SEelco Chaudron 186eac87c41SEelco Chaudron static void tbl_mask_array_reset_counters(struct mask_array *ma) 187eac87c41SEelco Chaudron { 188eac87c41SEelco Chaudron int i, cpu; 189eac87c41SEelco Chaudron 190eac87c41SEelco Chaudron /* As the per CPU counters are not atomic we can not go ahead and 191eac87c41SEelco Chaudron * reset them from another CPU. To be able to still have an approximate 192eac87c41SEelco Chaudron * zero based counter we store the value at reset, and subtract it 193eac87c41SEelco Chaudron * later when processing. 194eac87c41SEelco Chaudron */ 195eac87c41SEelco Chaudron for (i = 0; i < ma->max; i++) { 196eac87c41SEelco Chaudron ma->masks_usage_zero_cntr[i] = 0; 197eac87c41SEelco Chaudron 198eac87c41SEelco Chaudron for_each_possible_cpu(cpu) { 199eac87c41SEelco Chaudron u64 *usage_counters = per_cpu_ptr(ma->masks_usage_cntr, 200eac87c41SEelco Chaudron cpu); 201eac87c41SEelco Chaudron unsigned int start; 202eac87c41SEelco Chaudron u64 counter; 203eac87c41SEelco Chaudron 204eac87c41SEelco Chaudron do { 205eac87c41SEelco Chaudron start = u64_stats_fetch_begin_irq(&ma->syncp); 206eac87c41SEelco Chaudron counter = usage_counters[i]; 207eac87c41SEelco Chaudron } while (u64_stats_fetch_retry_irq(&ma->syncp, start)); 208eac87c41SEelco Chaudron 209eac87c41SEelco Chaudron ma->masks_usage_zero_cntr[i] += counter; 210eac87c41SEelco Chaudron } 211eac87c41SEelco Chaudron } 212eac87c41SEelco Chaudron } 213eac87c41SEelco Chaudron 2144bc63b1bSTonghao Zhang static struct mask_array *tbl_mask_array_alloc(int size) 2154bc63b1bSTonghao Zhang { 2164bc63b1bSTonghao Zhang struct mask_array *new; 2174bc63b1bSTonghao Zhang 2184bc63b1bSTonghao Zhang size = max(MASK_ARRAY_SIZE_MIN, size); 2194bc63b1bSTonghao Zhang new = kzalloc(sizeof(struct mask_array) + 220eac87c41SEelco Chaudron sizeof(struct sw_flow_mask *) * size + 221eac87c41SEelco Chaudron sizeof(u64) * size, GFP_KERNEL); 2224bc63b1bSTonghao Zhang if (!new) 2234bc63b1bSTonghao Zhang return NULL; 2244bc63b1bSTonghao Zhang 225eac87c41SEelco Chaudron new->masks_usage_zero_cntr = (u64 *)((u8 *)new + 226eac87c41SEelco Chaudron sizeof(struct mask_array) + 227eac87c41SEelco Chaudron sizeof(struct sw_flow_mask *) * 228eac87c41SEelco Chaudron size); 229eac87c41SEelco Chaudron 230eac87c41SEelco Chaudron new->masks_usage_cntr = __alloc_percpu(sizeof(u64) * size, 231eac87c41SEelco Chaudron __alignof__(u64)); 232eac87c41SEelco Chaudron if (!new->masks_usage_cntr) { 233eac87c41SEelco Chaudron kfree(new); 234eac87c41SEelco Chaudron return NULL; 235eac87c41SEelco Chaudron } 236eac87c41SEelco Chaudron 2374bc63b1bSTonghao Zhang new->count = 0; 2384bc63b1bSTonghao Zhang new->max = size; 2394bc63b1bSTonghao Zhang 2404bc63b1bSTonghao Zhang return new; 2414bc63b1bSTonghao Zhang } 2424bc63b1bSTonghao Zhang 2434bc63b1bSTonghao Zhang static int tbl_mask_array_realloc(struct flow_table *tbl, int size) 2444bc63b1bSTonghao Zhang { 2454bc63b1bSTonghao Zhang struct mask_array *old; 2464bc63b1bSTonghao Zhang struct mask_array *new; 2474bc63b1bSTonghao Zhang 2484bc63b1bSTonghao Zhang new = tbl_mask_array_alloc(size); 2494bc63b1bSTonghao Zhang if (!new) 2504bc63b1bSTonghao Zhang return -ENOMEM; 2514bc63b1bSTonghao Zhang 2524bc63b1bSTonghao Zhang old = ovsl_dereference(tbl->mask_array); 2534bc63b1bSTonghao Zhang if (old) { 2544bc63b1bSTonghao Zhang int i; 2554bc63b1bSTonghao Zhang 2564bc63b1bSTonghao Zhang for (i = 0; i < old->max; i++) { 2574bc63b1bSTonghao Zhang if (ovsl_dereference(old->masks[i])) 2584bc63b1bSTonghao Zhang new->masks[new->count++] = old->masks[i]; 2594bc63b1bSTonghao Zhang } 260eac87c41SEelco Chaudron call_rcu(&old->rcu, mask_array_rcu_cb); 2614bc63b1bSTonghao Zhang } 2624bc63b1bSTonghao Zhang 2634bc63b1bSTonghao Zhang rcu_assign_pointer(tbl->mask_array, new); 2644bc63b1bSTonghao Zhang 2654bc63b1bSTonghao Zhang return 0; 2664bc63b1bSTonghao Zhang } 2674bc63b1bSTonghao Zhang 26850b0e61bSTonghao Zhang static int tbl_mask_array_add_mask(struct flow_table *tbl, 26950b0e61bSTonghao Zhang struct sw_flow_mask *new) 27050b0e61bSTonghao Zhang { 27150b0e61bSTonghao Zhang struct mask_array *ma = ovsl_dereference(tbl->mask_array); 27250b0e61bSTonghao Zhang int err, ma_count = READ_ONCE(ma->count); 27350b0e61bSTonghao Zhang 27450b0e61bSTonghao Zhang if (ma_count >= ma->max) { 27550b0e61bSTonghao Zhang err = tbl_mask_array_realloc(tbl, ma->max + 27650b0e61bSTonghao Zhang MASK_ARRAY_SIZE_MIN); 27750b0e61bSTonghao Zhang if (err) 27850b0e61bSTonghao Zhang return err; 27950b0e61bSTonghao Zhang 28050b0e61bSTonghao Zhang ma = ovsl_dereference(tbl->mask_array); 281eac87c41SEelco Chaudron } else { 282eac87c41SEelco Chaudron /* On every add or delete we need to reset the counters so 283eac87c41SEelco Chaudron * every new mask gets a fair chance of being prioritized. 284eac87c41SEelco Chaudron */ 285eac87c41SEelco Chaudron tbl_mask_array_reset_counters(ma); 28650b0e61bSTonghao Zhang } 28750b0e61bSTonghao Zhang 28850b0e61bSTonghao Zhang BUG_ON(ovsl_dereference(ma->masks[ma_count])); 28950b0e61bSTonghao Zhang 29050b0e61bSTonghao Zhang rcu_assign_pointer(ma->masks[ma_count], new); 29150b0e61bSTonghao Zhang WRITE_ONCE(ma->count, ma_count +1); 29250b0e61bSTonghao Zhang 29350b0e61bSTonghao Zhang return 0; 29450b0e61bSTonghao Zhang } 29550b0e61bSTonghao Zhang 29650b0e61bSTonghao Zhang static void tbl_mask_array_del_mask(struct flow_table *tbl, 29750b0e61bSTonghao Zhang struct sw_flow_mask *mask) 29850b0e61bSTonghao Zhang { 29950b0e61bSTonghao Zhang struct mask_array *ma = ovsl_dereference(tbl->mask_array); 30050b0e61bSTonghao Zhang int i, ma_count = READ_ONCE(ma->count); 30150b0e61bSTonghao Zhang 30250b0e61bSTonghao Zhang /* Remove the deleted mask pointers from the array */ 30350b0e61bSTonghao Zhang for (i = 0; i < ma_count; i++) { 30450b0e61bSTonghao Zhang if (mask == ovsl_dereference(ma->masks[i])) 30550b0e61bSTonghao Zhang goto found; 30650b0e61bSTonghao Zhang } 30750b0e61bSTonghao Zhang 30850b0e61bSTonghao Zhang BUG(); 30950b0e61bSTonghao Zhang return; 31050b0e61bSTonghao Zhang 31150b0e61bSTonghao Zhang found: 31250b0e61bSTonghao Zhang WRITE_ONCE(ma->count, ma_count -1); 31350b0e61bSTonghao Zhang 31450b0e61bSTonghao Zhang rcu_assign_pointer(ma->masks[i], ma->masks[ma_count -1]); 31550b0e61bSTonghao Zhang RCU_INIT_POINTER(ma->masks[ma_count -1], NULL); 31650b0e61bSTonghao Zhang 31750b0e61bSTonghao Zhang kfree_rcu(mask, rcu); 31850b0e61bSTonghao Zhang 31950b0e61bSTonghao Zhang /* Shrink the mask array if necessary. */ 32050b0e61bSTonghao Zhang if (ma->max >= (MASK_ARRAY_SIZE_MIN * 2) && 32150b0e61bSTonghao Zhang ma_count <= (ma->max / 3)) 32250b0e61bSTonghao Zhang tbl_mask_array_realloc(tbl, ma->max / 2); 323eac87c41SEelco Chaudron else 324eac87c41SEelco Chaudron tbl_mask_array_reset_counters(ma); 325eac87c41SEelco Chaudron 32650b0e61bSTonghao Zhang } 32750b0e61bSTonghao Zhang 32850b0e61bSTonghao Zhang /* Remove 'mask' from the mask list, if it is not needed any more. */ 32950b0e61bSTonghao Zhang static void flow_mask_remove(struct flow_table *tbl, struct sw_flow_mask *mask) 33050b0e61bSTonghao Zhang { 33150b0e61bSTonghao Zhang if (mask) { 33250b0e61bSTonghao Zhang /* ovs-lock is required to protect mask-refcount and 33350b0e61bSTonghao Zhang * mask list. 33450b0e61bSTonghao Zhang */ 33550b0e61bSTonghao Zhang ASSERT_OVSL(); 33650b0e61bSTonghao Zhang BUG_ON(!mask->ref_count); 33750b0e61bSTonghao Zhang mask->ref_count--; 33850b0e61bSTonghao Zhang 33950b0e61bSTonghao Zhang if (!mask->ref_count) 34050b0e61bSTonghao Zhang tbl_mask_array_del_mask(tbl, mask); 34150b0e61bSTonghao Zhang } 34250b0e61bSTonghao Zhang } 34350b0e61bSTonghao Zhang 344b637e498SPravin B Shelar int ovs_flow_tbl_init(struct flow_table *table) 345b637e498SPravin B Shelar { 34674ed7ab9SJoe Stringer struct table_instance *ti, *ufid_ti; 3474bc63b1bSTonghao Zhang struct mask_array *ma; 348b637e498SPravin B Shelar 34904b7d136STonghao Zhang table->mask_cache = __alloc_percpu(sizeof(struct mask_cache_entry) * 35004b7d136STonghao Zhang MC_HASH_ENTRIES, 35104b7d136STonghao Zhang __alignof__(struct mask_cache_entry)); 35204b7d136STonghao Zhang if (!table->mask_cache) 353b637e498SPravin B Shelar return -ENOMEM; 354b637e498SPravin B Shelar 3554bc63b1bSTonghao Zhang ma = tbl_mask_array_alloc(MASK_ARRAY_SIZE_MIN); 3564bc63b1bSTonghao Zhang if (!ma) 3574bc63b1bSTonghao Zhang goto free_mask_cache; 3584bc63b1bSTonghao Zhang 35904b7d136STonghao Zhang ti = table_instance_alloc(TBL_MIN_BUCKETS); 36004b7d136STonghao Zhang if (!ti) 3614bc63b1bSTonghao Zhang goto free_mask_array; 36204b7d136STonghao Zhang 36374ed7ab9SJoe Stringer ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS); 36474ed7ab9SJoe Stringer if (!ufid_ti) 36574ed7ab9SJoe Stringer goto free_ti; 36674ed7ab9SJoe Stringer 367b637e498SPravin B Shelar rcu_assign_pointer(table->ti, ti); 36874ed7ab9SJoe Stringer rcu_assign_pointer(table->ufid_ti, ufid_ti); 3694bc63b1bSTonghao Zhang rcu_assign_pointer(table->mask_array, ma); 370b637e498SPravin B Shelar table->last_rehash = jiffies; 371e6445719SPravin B Shelar table->count = 0; 37274ed7ab9SJoe Stringer table->ufid_count = 0; 373b637e498SPravin B Shelar return 0; 37474ed7ab9SJoe Stringer 37574ed7ab9SJoe Stringer free_ti: 37674ed7ab9SJoe Stringer __table_instance_destroy(ti); 3774bc63b1bSTonghao Zhang free_mask_array: 378eac87c41SEelco Chaudron __mask_array_destroy(ma); 37904b7d136STonghao Zhang free_mask_cache: 38004b7d136STonghao Zhang free_percpu(table->mask_cache); 38174ed7ab9SJoe Stringer return -ENOMEM; 382e6445719SPravin B Shelar } 383e6445719SPravin B Shelar 384e6445719SPravin B Shelar static void flow_tbl_destroy_rcu_cb(struct rcu_head *rcu) 385e6445719SPravin B Shelar { 386b637e498SPravin B Shelar struct table_instance *ti = container_of(rcu, struct table_instance, rcu); 387e6445719SPravin B Shelar 388b637e498SPravin B Shelar __table_instance_destroy(ti); 389b637e498SPravin B Shelar } 390b637e498SPravin B Shelar 39150b0e61bSTonghao Zhang static void table_instance_flow_free(struct flow_table *table, 39250b0e61bSTonghao Zhang struct table_instance *ti, 39350b0e61bSTonghao Zhang struct table_instance *ufid_ti, 39450b0e61bSTonghao Zhang struct sw_flow *flow, 39550b0e61bSTonghao Zhang bool count) 39650b0e61bSTonghao Zhang { 39750b0e61bSTonghao Zhang hlist_del_rcu(&flow->flow_table.node[ti->node_ver]); 39850b0e61bSTonghao Zhang if (count) 39950b0e61bSTonghao Zhang table->count--; 40050b0e61bSTonghao Zhang 40150b0e61bSTonghao Zhang if (ovs_identifier_is_ufid(&flow->id)) { 40250b0e61bSTonghao Zhang hlist_del_rcu(&flow->ufid_table.node[ufid_ti->node_ver]); 40350b0e61bSTonghao Zhang 40450b0e61bSTonghao Zhang if (count) 40550b0e61bSTonghao Zhang table->ufid_count--; 40650b0e61bSTonghao Zhang } 40750b0e61bSTonghao Zhang 40850b0e61bSTonghao Zhang flow_mask_remove(table, flow->mask); 40950b0e61bSTonghao Zhang } 41050b0e61bSTonghao Zhang 41150b0e61bSTonghao Zhang static void table_instance_destroy(struct flow_table *table, 41250b0e61bSTonghao Zhang struct table_instance *ti, 41374ed7ab9SJoe Stringer struct table_instance *ufid_ti, 41474ed7ab9SJoe Stringer bool deferred) 415b637e498SPravin B Shelar { 416e80857ccSAndy Zhou int i; 417e80857ccSAndy Zhou 418b637e498SPravin B Shelar if (!ti) 419b637e498SPravin B Shelar return; 420b637e498SPravin B Shelar 42174ed7ab9SJoe Stringer BUG_ON(!ufid_ti); 422e80857ccSAndy Zhou if (ti->keep_flows) 423e80857ccSAndy Zhou goto skip_flows; 424e80857ccSAndy Zhou 425e80857ccSAndy Zhou for (i = 0; i < ti->n_buckets; i++) { 426e80857ccSAndy Zhou struct sw_flow *flow; 427ee9c5e67SKent Overstreet struct hlist_head *head = &ti->buckets[i]; 428e80857ccSAndy Zhou struct hlist_node *n; 429e80857ccSAndy Zhou 43050b0e61bSTonghao Zhang hlist_for_each_entry_safe(flow, n, head, 43150b0e61bSTonghao Zhang flow_table.node[ti->node_ver]) { 43250b0e61bSTonghao Zhang 43350b0e61bSTonghao Zhang table_instance_flow_free(table, ti, ufid_ti, 43450b0e61bSTonghao Zhang flow, false); 435e80857ccSAndy Zhou ovs_flow_free(flow, deferred); 436e80857ccSAndy Zhou } 437e80857ccSAndy Zhou } 438e80857ccSAndy Zhou 439e80857ccSAndy Zhou skip_flows: 44074ed7ab9SJoe Stringer if (deferred) { 441b637e498SPravin B Shelar call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb); 44274ed7ab9SJoe Stringer call_rcu(&ufid_ti->rcu, flow_tbl_destroy_rcu_cb); 44374ed7ab9SJoe Stringer } else { 444b637e498SPravin B Shelar __table_instance_destroy(ti); 44574ed7ab9SJoe Stringer __table_instance_destroy(ufid_ti); 44674ed7ab9SJoe Stringer } 447e6445719SPravin B Shelar } 448e6445719SPravin B Shelar 4499b996e54SPravin B Shelar /* No need for locking this function is called from RCU callback or 4509b996e54SPravin B Shelar * error path. 4519b996e54SPravin B Shelar */ 4529b996e54SPravin B Shelar void ovs_flow_tbl_destroy(struct flow_table *table) 453e6445719SPravin B Shelar { 4549b996e54SPravin B Shelar struct table_instance *ti = rcu_dereference_raw(table->ti); 45574ed7ab9SJoe Stringer struct table_instance *ufid_ti = rcu_dereference_raw(table->ufid_ti); 456e6445719SPravin B Shelar 45704b7d136STonghao Zhang free_percpu(table->mask_cache); 458eac87c41SEelco Chaudron call_rcu(&table->mask_array->rcu, mask_array_rcu_cb); 45950b0e61bSTonghao Zhang table_instance_destroy(table, ti, ufid_ti, false); 460e6445719SPravin B Shelar } 461e6445719SPravin B Shelar 462b637e498SPravin B Shelar struct sw_flow *ovs_flow_tbl_dump_next(struct table_instance *ti, 463e6445719SPravin B Shelar u32 *bucket, u32 *last) 464e6445719SPravin B Shelar { 465e6445719SPravin B Shelar struct sw_flow *flow; 466e6445719SPravin B Shelar struct hlist_head *head; 467e6445719SPravin B Shelar int ver; 468e6445719SPravin B Shelar int i; 469e6445719SPravin B Shelar 470b637e498SPravin B Shelar ver = ti->node_ver; 471b637e498SPravin B Shelar while (*bucket < ti->n_buckets) { 472e6445719SPravin B Shelar i = 0; 473ee9c5e67SKent Overstreet head = &ti->buckets[*bucket]; 47474ed7ab9SJoe Stringer hlist_for_each_entry_rcu(flow, head, flow_table.node[ver]) { 475e6445719SPravin B Shelar if (i < *last) { 476e6445719SPravin B Shelar i++; 477e6445719SPravin B Shelar continue; 478e6445719SPravin B Shelar } 479e6445719SPravin B Shelar *last = i + 1; 480e6445719SPravin B Shelar return flow; 481e6445719SPravin B Shelar } 482e6445719SPravin B Shelar (*bucket)++; 483e6445719SPravin B Shelar *last = 0; 484e6445719SPravin B Shelar } 485e6445719SPravin B Shelar 486e6445719SPravin B Shelar return NULL; 487e6445719SPravin B Shelar } 488e6445719SPravin B Shelar 489b637e498SPravin B Shelar static struct hlist_head *find_bucket(struct table_instance *ti, u32 hash) 490e6445719SPravin B Shelar { 491b637e498SPravin B Shelar hash = jhash_1word(hash, ti->hash_seed); 492ee9c5e67SKent Overstreet return &ti->buckets[hash & (ti->n_buckets - 1)]; 493e6445719SPravin B Shelar } 494e6445719SPravin B Shelar 49574ed7ab9SJoe Stringer static void table_instance_insert(struct table_instance *ti, 49674ed7ab9SJoe Stringer struct sw_flow *flow) 497e6445719SPravin B Shelar { 498e6445719SPravin B Shelar struct hlist_head *head; 499e6445719SPravin B Shelar 50074ed7ab9SJoe Stringer head = find_bucket(ti, flow->flow_table.hash); 50174ed7ab9SJoe Stringer hlist_add_head_rcu(&flow->flow_table.node[ti->node_ver], head); 50274ed7ab9SJoe Stringer } 50374ed7ab9SJoe Stringer 50474ed7ab9SJoe Stringer static void ufid_table_instance_insert(struct table_instance *ti, 50574ed7ab9SJoe Stringer struct sw_flow *flow) 50674ed7ab9SJoe Stringer { 50774ed7ab9SJoe Stringer struct hlist_head *head; 50874ed7ab9SJoe Stringer 50974ed7ab9SJoe Stringer head = find_bucket(ti, flow->ufid_table.hash); 51074ed7ab9SJoe Stringer hlist_add_head_rcu(&flow->ufid_table.node[ti->node_ver], head); 511e6445719SPravin B Shelar } 512e6445719SPravin B Shelar 513b637e498SPravin B Shelar static void flow_table_copy_flows(struct table_instance *old, 51474ed7ab9SJoe Stringer struct table_instance *new, bool ufid) 515e6445719SPravin B Shelar { 516e6445719SPravin B Shelar int old_ver; 517e6445719SPravin B Shelar int i; 518e6445719SPravin B Shelar 519e6445719SPravin B Shelar old_ver = old->node_ver; 520e6445719SPravin B Shelar new->node_ver = !old_ver; 521e6445719SPravin B Shelar 522e6445719SPravin B Shelar /* Insert in new table. */ 523e6445719SPravin B Shelar for (i = 0; i < old->n_buckets; i++) { 524e6445719SPravin B Shelar struct sw_flow *flow; 525ee9c5e67SKent Overstreet struct hlist_head *head = &old->buckets[i]; 526e6445719SPravin B Shelar 52774ed7ab9SJoe Stringer if (ufid) 52864948427STonghao Zhang hlist_for_each_entry_rcu(flow, head, 52964948427STonghao Zhang ufid_table.node[old_ver], 53064948427STonghao Zhang lockdep_ovsl_is_held()) 53174ed7ab9SJoe Stringer ufid_table_instance_insert(new, flow); 53274ed7ab9SJoe Stringer else 53364948427STonghao Zhang hlist_for_each_entry_rcu(flow, head, 53464948427STonghao Zhang flow_table.node[old_ver], 53564948427STonghao Zhang lockdep_ovsl_is_held()) 536b637e498SPravin B Shelar table_instance_insert(new, flow); 537e6445719SPravin B Shelar } 538e6445719SPravin B Shelar 539e6445719SPravin B Shelar old->keep_flows = true; 540e6445719SPravin B Shelar } 541e6445719SPravin B Shelar 542b637e498SPravin B Shelar static struct table_instance *table_instance_rehash(struct table_instance *ti, 54374ed7ab9SJoe Stringer int n_buckets, bool ufid) 544e6445719SPravin B Shelar { 545b637e498SPravin B Shelar struct table_instance *new_ti; 546e6445719SPravin B Shelar 547b637e498SPravin B Shelar new_ti = table_instance_alloc(n_buckets); 548b637e498SPravin B Shelar if (!new_ti) 549618ed0c8SPravin B Shelar return NULL; 550e6445719SPravin B Shelar 55174ed7ab9SJoe Stringer flow_table_copy_flows(ti, new_ti, ufid); 552e6445719SPravin B Shelar 553b637e498SPravin B Shelar return new_ti; 554e6445719SPravin B Shelar } 555e6445719SPravin B Shelar 556b637e498SPravin B Shelar int ovs_flow_tbl_flush(struct flow_table *flow_table) 557e6445719SPravin B Shelar { 55874ed7ab9SJoe Stringer struct table_instance *old_ti, *new_ti; 55974ed7ab9SJoe Stringer struct table_instance *old_ufid_ti, *new_ufid_ti; 560e6445719SPravin B Shelar 561b637e498SPravin B Shelar new_ti = table_instance_alloc(TBL_MIN_BUCKETS); 562b637e498SPravin B Shelar if (!new_ti) 563b637e498SPravin B Shelar return -ENOMEM; 56474ed7ab9SJoe Stringer new_ufid_ti = table_instance_alloc(TBL_MIN_BUCKETS); 56574ed7ab9SJoe Stringer if (!new_ufid_ti) 56674ed7ab9SJoe Stringer goto err_free_ti; 56774ed7ab9SJoe Stringer 56874ed7ab9SJoe Stringer old_ti = ovsl_dereference(flow_table->ti); 56974ed7ab9SJoe Stringer old_ufid_ti = ovsl_dereference(flow_table->ufid_ti); 570b637e498SPravin B Shelar 571b637e498SPravin B Shelar rcu_assign_pointer(flow_table->ti, new_ti); 57274ed7ab9SJoe Stringer rcu_assign_pointer(flow_table->ufid_ti, new_ufid_ti); 573b637e498SPravin B Shelar flow_table->last_rehash = jiffies; 574b637e498SPravin B Shelar flow_table->count = 0; 57574ed7ab9SJoe Stringer flow_table->ufid_count = 0; 576b637e498SPravin B Shelar 57750b0e61bSTonghao Zhang table_instance_destroy(flow_table, old_ti, old_ufid_ti, true); 578b637e498SPravin B Shelar return 0; 57974ed7ab9SJoe Stringer 58074ed7ab9SJoe Stringer err_free_ti: 58174ed7ab9SJoe Stringer __table_instance_destroy(new_ti); 58274ed7ab9SJoe Stringer return -ENOMEM; 583e6445719SPravin B Shelar } 584e6445719SPravin B Shelar 585272c2cf8SJoe Stringer static u32 flow_hash(const struct sw_flow_key *key, 586272c2cf8SJoe Stringer const struct sw_flow_key_range *range) 587e6445719SPravin B Shelar { 588515b65a4STonghao Zhang const u32 *hash_key = (const u32 *)((const u8 *)key + range->start); 589e6445719SPravin B Shelar 590e6445719SPravin B Shelar /* Make sure number of hash bytes are multiple of u32. */ 591515b65a4STonghao Zhang int hash_u32s = range_n_bytes(range) >> 2; 592e6445719SPravin B Shelar 59387545899SDaniel Borkmann return jhash2(hash_key, hash_u32s, 0); 594e6445719SPravin B Shelar } 595e6445719SPravin B Shelar 596e6445719SPravin B Shelar static int flow_key_start(const struct sw_flow_key *key) 597e6445719SPravin B Shelar { 59800a93babSJiri Benc if (key->tun_proto) 599e6445719SPravin B Shelar return 0; 600e6445719SPravin B Shelar else 601e6445719SPravin B Shelar return rounddown(offsetof(struct sw_flow_key, phy), 602e6445719SPravin B Shelar sizeof(long)); 603e6445719SPravin B Shelar } 604e6445719SPravin B Shelar 605e6445719SPravin B Shelar static bool cmp_key(const struct sw_flow_key *key1, 606e6445719SPravin B Shelar const struct sw_flow_key *key2, 607e6445719SPravin B Shelar int key_start, int key_end) 608e6445719SPravin B Shelar { 6097085130bSDaniele Di Proietto const long *cp1 = (const long *)((const u8 *)key1 + key_start); 6107085130bSDaniele Di Proietto const long *cp2 = (const long *)((const u8 *)key2 + key_start); 611e6445719SPravin B Shelar long diffs = 0; 612e6445719SPravin B Shelar int i; 613e6445719SPravin B Shelar 614e6445719SPravin B Shelar for (i = key_start; i < key_end; i += sizeof(long)) 615e6445719SPravin B Shelar diffs |= *cp1++ ^ *cp2++; 616e6445719SPravin B Shelar 617e6445719SPravin B Shelar return diffs == 0; 618e6445719SPravin B Shelar } 619e6445719SPravin B Shelar 620e6445719SPravin B Shelar static bool flow_cmp_masked_key(const struct sw_flow *flow, 621e6445719SPravin B Shelar const struct sw_flow_key *key, 622272c2cf8SJoe Stringer const struct sw_flow_key_range *range) 623e6445719SPravin B Shelar { 624272c2cf8SJoe Stringer return cmp_key(&flow->key, key, range->start, range->end); 625e6445719SPravin B Shelar } 626e6445719SPravin B Shelar 62774ed7ab9SJoe Stringer static bool ovs_flow_cmp_unmasked_key(const struct sw_flow *flow, 62812eb18f7SThomas Graf const struct sw_flow_match *match) 629e6445719SPravin B Shelar { 630e6445719SPravin B Shelar struct sw_flow_key *key = match->key; 631e6445719SPravin B Shelar int key_start = flow_key_start(key); 632e6445719SPravin B Shelar int key_end = match->range.end; 633e6445719SPravin B Shelar 63474ed7ab9SJoe Stringer BUG_ON(ovs_identifier_is_ufid(&flow->id)); 63574ed7ab9SJoe Stringer return cmp_key(flow->id.unmasked_key, key, key_start, key_end); 636e6445719SPravin B Shelar } 637e6445719SPravin B Shelar 638b637e498SPravin B Shelar static struct sw_flow *masked_flow_lookup(struct table_instance *ti, 639e6445719SPravin B Shelar const struct sw_flow_key *unmasked, 64004b7d136STonghao Zhang const struct sw_flow_mask *mask, 64104b7d136STonghao Zhang u32 *n_mask_hit) 642e6445719SPravin B Shelar { 643e6445719SPravin B Shelar struct sw_flow *flow; 644e6445719SPravin B Shelar struct hlist_head *head; 645e6445719SPravin B Shelar u32 hash; 646e6445719SPravin B Shelar struct sw_flow_key masked_key; 647e6445719SPravin B Shelar 648ae5f2fb1SJesse Gross ovs_flow_mask_key(&masked_key, unmasked, false, mask); 649272c2cf8SJoe Stringer hash = flow_hash(&masked_key, &mask->range); 650b637e498SPravin B Shelar head = find_bucket(ti, hash); 65104b7d136STonghao Zhang (*n_mask_hit)++; 65204b7d136STonghao Zhang 653a2cfb96cSMadhuparna Bhowmik hlist_for_each_entry_rcu(flow, head, flow_table.node[ti->node_ver], 654a2cfb96cSMadhuparna Bhowmik lockdep_ovsl_is_held()) { 65574ed7ab9SJoe Stringer if (flow->mask == mask && flow->flow_table.hash == hash && 656272c2cf8SJoe Stringer flow_cmp_masked_key(flow, &masked_key, &mask->range)) 657e6445719SPravin B Shelar return flow; 658e6445719SPravin B Shelar } 659e6445719SPravin B Shelar return NULL; 660e6445719SPravin B Shelar } 661e6445719SPravin B Shelar 662a7f35e78STonghao Zhang /* Flow lookup does full lookup on flow table. It starts with 663a7f35e78STonghao Zhang * mask from index passed in *index. 664a7f35e78STonghao Zhang */ 66504b7d136STonghao Zhang static struct sw_flow *flow_lookup(struct flow_table *tbl, 66604b7d136STonghao Zhang struct table_instance *ti, 6674bc63b1bSTonghao Zhang struct mask_array *ma, 6681bd7116fSAndy Zhou const struct sw_flow_key *key, 6694bc63b1bSTonghao Zhang u32 *n_mask_hit, 670*9d2f627bSEelco Chaudron u32 *n_cache_hit, 6714bc63b1bSTonghao Zhang u32 *index) 672e6445719SPravin B Shelar { 673eac87c41SEelco Chaudron u64 *usage_counters = this_cpu_ptr(ma->masks_usage_cntr); 674b637e498SPravin B Shelar struct sw_flow *flow; 67557f7d7b9STonghao Zhang struct sw_flow_mask *mask; 6764bc63b1bSTonghao Zhang int i; 677e6445719SPravin B Shelar 6780a3e0137STonghao Zhang if (likely(*index < ma->max)) { 679a7f35e78STonghao Zhang mask = rcu_dereference_ovsl(ma->masks[*index]); 680a7f35e78STonghao Zhang if (mask) { 681a7f35e78STonghao Zhang flow = masked_flow_lookup(ti, key, mask, n_mask_hit); 682eac87c41SEelco Chaudron if (flow) { 683eac87c41SEelco Chaudron u64_stats_update_begin(&ma->syncp); 684eac87c41SEelco Chaudron usage_counters[*index]++; 685eac87c41SEelco Chaudron u64_stats_update_end(&ma->syncp); 686*9d2f627bSEelco Chaudron (*n_cache_hit)++; 687a7f35e78STonghao Zhang return flow; 688a7f35e78STonghao Zhang } 689a7f35e78STonghao Zhang } 690eac87c41SEelco Chaudron } 691a7f35e78STonghao Zhang 6924bc63b1bSTonghao Zhang for (i = 0; i < ma->max; i++) { 693a7f35e78STonghao Zhang 694a7f35e78STonghao Zhang if (i == *index) 695a7f35e78STonghao Zhang continue; 6964bc63b1bSTonghao Zhang 6974bc63b1bSTonghao Zhang mask = rcu_dereference_ovsl(ma->masks[i]); 6980a3e0137STonghao Zhang if (unlikely(!mask)) 69957f7d7b9STonghao Zhang break; 700a7f35e78STonghao Zhang 70104b7d136STonghao Zhang flow = masked_flow_lookup(ti, key, mask, n_mask_hit); 7024bc63b1bSTonghao Zhang if (flow) { /* Found */ 7034bc63b1bSTonghao Zhang *index = i; 704eac87c41SEelco Chaudron u64_stats_update_begin(&ma->syncp); 705eac87c41SEelco Chaudron usage_counters[*index]++; 706eac87c41SEelco Chaudron u64_stats_update_end(&ma->syncp); 707b637e498SPravin B Shelar return flow; 708b637e498SPravin B Shelar } 7094bc63b1bSTonghao Zhang } 7104bc63b1bSTonghao Zhang 711b637e498SPravin B Shelar return NULL; 712e6445719SPravin B Shelar } 713e6445719SPravin B Shelar 71404b7d136STonghao Zhang /* 71504b7d136STonghao Zhang * mask_cache maps flow to probable mask. This cache is not tightly 71604b7d136STonghao Zhang * coupled cache, It means updates to mask list can result in inconsistent 71704b7d136STonghao Zhang * cache entry in mask cache. 71804b7d136STonghao Zhang * This is per cpu cache and is divided in MC_HASH_SEGS segments. 71904b7d136STonghao Zhang * In case of a hash collision the entry is hashed in next segment. 72004b7d136STonghao Zhang * */ 72104b7d136STonghao Zhang struct sw_flow *ovs_flow_tbl_lookup_stats(struct flow_table *tbl, 72204b7d136STonghao Zhang const struct sw_flow_key *key, 72304b7d136STonghao Zhang u32 skb_hash, 724*9d2f627bSEelco Chaudron u32 *n_mask_hit, 725*9d2f627bSEelco Chaudron u32 *n_cache_hit) 72604b7d136STonghao Zhang { 727a7f35e78STonghao Zhang struct mask_array *ma = rcu_dereference(tbl->mask_array); 728a7f35e78STonghao Zhang struct table_instance *ti = rcu_dereference(tbl->ti); 729a7f35e78STonghao Zhang struct mask_cache_entry *entries, *ce; 73004b7d136STonghao Zhang struct sw_flow *flow; 731a7f35e78STonghao Zhang u32 hash; 73204b7d136STonghao Zhang int seg; 73304b7d136STonghao Zhang 73404b7d136STonghao Zhang *n_mask_hit = 0; 735*9d2f627bSEelco Chaudron *n_cache_hit = 0; 7364bc63b1bSTonghao Zhang if (unlikely(!skb_hash)) { 737a7f35e78STonghao Zhang u32 mask_index = 0; 738*9d2f627bSEelco Chaudron u32 cache = 0; 7394bc63b1bSTonghao Zhang 740*9d2f627bSEelco Chaudron return flow_lookup(tbl, ti, ma, key, n_mask_hit, &cache, 741*9d2f627bSEelco Chaudron &mask_index); 7424bc63b1bSTonghao Zhang } 74304b7d136STonghao Zhang 744a7f35e78STonghao Zhang /* Pre and post recirulation flows usually have the same skb_hash 745a7f35e78STonghao Zhang * value. To avoid hash collisions, rehash the 'skb_hash' with 746a7f35e78STonghao Zhang * 'recirc_id'. */ 747a7f35e78STonghao Zhang if (key->recirc_id) 748a7f35e78STonghao Zhang skb_hash = jhash_1word(skb_hash, key->recirc_id); 749a7f35e78STonghao Zhang 750a7f35e78STonghao Zhang ce = NULL; 751a7f35e78STonghao Zhang hash = skb_hash; 75204b7d136STonghao Zhang entries = this_cpu_ptr(tbl->mask_cache); 75304b7d136STonghao Zhang 754a7f35e78STonghao Zhang /* Find the cache entry 'ce' to operate on. */ 75504b7d136STonghao Zhang for (seg = 0; seg < MC_HASH_SEGS; seg++) { 756a7f35e78STonghao Zhang int index = hash & (MC_HASH_ENTRIES - 1); 757a7f35e78STonghao Zhang struct mask_cache_entry *e; 75804b7d136STonghao Zhang 759a7f35e78STonghao Zhang e = &entries[index]; 760a7f35e78STonghao Zhang if (e->skb_hash == skb_hash) { 761a7f35e78STonghao Zhang flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, 762*9d2f627bSEelco Chaudron n_cache_hit, &e->mask_index); 763a7f35e78STonghao Zhang if (!flow) 764a7f35e78STonghao Zhang e->skb_hash = 0; 76504b7d136STonghao Zhang return flow; 76604b7d136STonghao Zhang } 76704b7d136STonghao Zhang 768a7f35e78STonghao Zhang if (!ce || e->skb_hash < ce->skb_hash) 769a7f35e78STonghao Zhang ce = e; /* A better replacement cache candidate. */ 77004b7d136STonghao Zhang 77104b7d136STonghao Zhang hash >>= MC_HASH_SHIFT; 77204b7d136STonghao Zhang } 77304b7d136STonghao Zhang 774a7f35e78STonghao Zhang /* Cache miss, do full lookup. */ 775*9d2f627bSEelco Chaudron flow = flow_lookup(tbl, ti, ma, key, n_mask_hit, n_cache_hit, 776*9d2f627bSEelco Chaudron &ce->mask_index); 7774bc63b1bSTonghao Zhang if (flow) 778a7f35e78STonghao Zhang ce->skb_hash = skb_hash; 77904b7d136STonghao Zhang 780*9d2f627bSEelco Chaudron *n_cache_hit = 0; 78104b7d136STonghao Zhang return flow; 78204b7d136STonghao Zhang } 78304b7d136STonghao Zhang 7845bb50632SAndy Zhou struct sw_flow *ovs_flow_tbl_lookup(struct flow_table *tbl, 7855bb50632SAndy Zhou const struct sw_flow_key *key) 7865bb50632SAndy Zhou { 78704b7d136STonghao Zhang struct table_instance *ti = rcu_dereference_ovsl(tbl->ti); 7884bc63b1bSTonghao Zhang struct mask_array *ma = rcu_dereference_ovsl(tbl->mask_array); 7894bc63b1bSTonghao Zhang u32 __always_unused n_mask_hit; 790*9d2f627bSEelco Chaudron u32 __always_unused n_cache_hit; 791a7f35e78STonghao Zhang u32 index = 0; 7924bc63b1bSTonghao Zhang 793*9d2f627bSEelco Chaudron return flow_lookup(tbl, ti, ma, key, &n_mask_hit, &n_cache_hit, &index); 7945bb50632SAndy Zhou } 7955bb50632SAndy Zhou 7964a46b24eSAlex Wang struct sw_flow *ovs_flow_tbl_lookup_exact(struct flow_table *tbl, 79712eb18f7SThomas Graf const struct sw_flow_match *match) 7984a46b24eSAlex Wang { 7994bc63b1bSTonghao Zhang struct mask_array *ma = ovsl_dereference(tbl->mask_array); 8004bc63b1bSTonghao Zhang int i; 8014a46b24eSAlex Wang 8024a46b24eSAlex Wang /* Always called under ovs-mutex. */ 8034bc63b1bSTonghao Zhang for (i = 0; i < ma->max; i++) { 8044bc63b1bSTonghao Zhang struct table_instance *ti = rcu_dereference_ovsl(tbl->ti); 8054bc63b1bSTonghao Zhang u32 __always_unused n_mask_hit; 8064bc63b1bSTonghao Zhang struct sw_flow_mask *mask; 8074bc63b1bSTonghao Zhang struct sw_flow *flow; 8084bc63b1bSTonghao Zhang 8094bc63b1bSTonghao Zhang mask = ovsl_dereference(ma->masks[i]); 8104bc63b1bSTonghao Zhang if (!mask) 8114bc63b1bSTonghao Zhang continue; 8124bc63b1bSTonghao Zhang 81304b7d136STonghao Zhang flow = masked_flow_lookup(ti, match->key, mask, &n_mask_hit); 81474ed7ab9SJoe Stringer if (flow && ovs_identifier_is_key(&flow->id) && 8154bc63b1bSTonghao Zhang ovs_flow_cmp_unmasked_key(flow, match)) { 81674ed7ab9SJoe Stringer return flow; 81774ed7ab9SJoe Stringer } 8184bc63b1bSTonghao Zhang } 8194bc63b1bSTonghao Zhang 82074ed7ab9SJoe Stringer return NULL; 82174ed7ab9SJoe Stringer } 82274ed7ab9SJoe Stringer 82374ed7ab9SJoe Stringer static u32 ufid_hash(const struct sw_flow_id *sfid) 82474ed7ab9SJoe Stringer { 82574ed7ab9SJoe Stringer return jhash(sfid->ufid, sfid->ufid_len, 0); 82674ed7ab9SJoe Stringer } 82774ed7ab9SJoe Stringer 82874ed7ab9SJoe Stringer static bool ovs_flow_cmp_ufid(const struct sw_flow *flow, 82974ed7ab9SJoe Stringer const struct sw_flow_id *sfid) 83074ed7ab9SJoe Stringer { 83174ed7ab9SJoe Stringer if (flow->id.ufid_len != sfid->ufid_len) 83274ed7ab9SJoe Stringer return false; 83374ed7ab9SJoe Stringer 83474ed7ab9SJoe Stringer return !memcmp(flow->id.ufid, sfid->ufid, sfid->ufid_len); 83574ed7ab9SJoe Stringer } 83674ed7ab9SJoe Stringer 83774ed7ab9SJoe Stringer bool ovs_flow_cmp(const struct sw_flow *flow, const struct sw_flow_match *match) 83874ed7ab9SJoe Stringer { 83974ed7ab9SJoe Stringer if (ovs_identifier_is_ufid(&flow->id)) 84074ed7ab9SJoe Stringer return flow_cmp_masked_key(flow, match->key, &match->range); 84174ed7ab9SJoe Stringer 84274ed7ab9SJoe Stringer return ovs_flow_cmp_unmasked_key(flow, match); 84374ed7ab9SJoe Stringer } 84474ed7ab9SJoe Stringer 84574ed7ab9SJoe Stringer struct sw_flow *ovs_flow_tbl_lookup_ufid(struct flow_table *tbl, 84674ed7ab9SJoe Stringer const struct sw_flow_id *ufid) 84774ed7ab9SJoe Stringer { 84874ed7ab9SJoe Stringer struct table_instance *ti = rcu_dereference_ovsl(tbl->ufid_ti); 84974ed7ab9SJoe Stringer struct sw_flow *flow; 85074ed7ab9SJoe Stringer struct hlist_head *head; 85174ed7ab9SJoe Stringer u32 hash; 85274ed7ab9SJoe Stringer 85374ed7ab9SJoe Stringer hash = ufid_hash(ufid); 85474ed7ab9SJoe Stringer head = find_bucket(ti, hash); 855a2cfb96cSMadhuparna Bhowmik hlist_for_each_entry_rcu(flow, head, ufid_table.node[ti->node_ver], 856a2cfb96cSMadhuparna Bhowmik lockdep_ovsl_is_held()) { 85774ed7ab9SJoe Stringer if (flow->ufid_table.hash == hash && 85874ed7ab9SJoe Stringer ovs_flow_cmp_ufid(flow, ufid)) 8594a46b24eSAlex Wang return flow; 8604a46b24eSAlex Wang } 8614a46b24eSAlex Wang return NULL; 8624a46b24eSAlex Wang } 8634a46b24eSAlex Wang 8641bd7116fSAndy Zhou int ovs_flow_tbl_num_masks(const struct flow_table *table) 8651bd7116fSAndy Zhou { 8664bc63b1bSTonghao Zhang struct mask_array *ma = rcu_dereference_ovsl(table->mask_array); 86757f7d7b9STonghao Zhang return READ_ONCE(ma->count); 8681bd7116fSAndy Zhou } 8691bd7116fSAndy Zhou 87074ed7ab9SJoe Stringer static struct table_instance *table_instance_expand(struct table_instance *ti, 87174ed7ab9SJoe Stringer bool ufid) 872b637e498SPravin B Shelar { 87374ed7ab9SJoe Stringer return table_instance_rehash(ti, ti->n_buckets * 2, ufid); 874e6445719SPravin B Shelar } 875e6445719SPravin B Shelar 87656c19868SJarno Rajahalme /* Must be called with OVS mutex held. */ 877e6445719SPravin B Shelar void ovs_flow_tbl_remove(struct flow_table *table, struct sw_flow *flow) 878e6445719SPravin B Shelar { 879b637e498SPravin B Shelar struct table_instance *ti = ovsl_dereference(table->ti); 88074ed7ab9SJoe Stringer struct table_instance *ufid_ti = ovsl_dereference(table->ufid_ti); 881b637e498SPravin B Shelar 882e6445719SPravin B Shelar BUG_ON(table->count == 0); 88350b0e61bSTonghao Zhang table_instance_flow_free(table, ti, ufid_ti, flow, true); 884e6445719SPravin B Shelar } 885e6445719SPravin B Shelar 886618ed0c8SPravin B Shelar static struct sw_flow_mask *mask_alloc(void) 887e6445719SPravin B Shelar { 888e6445719SPravin B Shelar struct sw_flow_mask *mask; 889e6445719SPravin B Shelar 890e6445719SPravin B Shelar mask = kmalloc(sizeof(*mask), GFP_KERNEL); 891e6445719SPravin B Shelar if (mask) 892e80857ccSAndy Zhou mask->ref_count = 1; 893e6445719SPravin B Shelar 894e6445719SPravin B Shelar return mask; 895e6445719SPravin B Shelar } 896e6445719SPravin B Shelar 897e6445719SPravin B Shelar static bool mask_equal(const struct sw_flow_mask *a, 898e6445719SPravin B Shelar const struct sw_flow_mask *b) 899e6445719SPravin B Shelar { 9007085130bSDaniele Di Proietto const u8 *a_ = (const u8 *)&a->key + a->range.start; 9017085130bSDaniele Di Proietto const u8 *b_ = (const u8 *)&b->key + b->range.start; 902e6445719SPravin B Shelar 903e6445719SPravin B Shelar return (a->range.end == b->range.end) 904e6445719SPravin B Shelar && (a->range.start == b->range.start) 905e6445719SPravin B Shelar && (memcmp(a_, b_, range_n_bytes(&a->range)) == 0); 906e6445719SPravin B Shelar } 907e6445719SPravin B Shelar 908618ed0c8SPravin B Shelar static struct sw_flow_mask *flow_mask_find(const struct flow_table *tbl, 909e6445719SPravin B Shelar const struct sw_flow_mask *mask) 910e6445719SPravin B Shelar { 9114bc63b1bSTonghao Zhang struct mask_array *ma; 9124bc63b1bSTonghao Zhang int i; 913e6445719SPravin B Shelar 9144bc63b1bSTonghao Zhang ma = ovsl_dereference(tbl->mask_array); 9154bc63b1bSTonghao Zhang for (i = 0; i < ma->max; i++) { 9164bc63b1bSTonghao Zhang struct sw_flow_mask *t; 9174bc63b1bSTonghao Zhang t = ovsl_dereference(ma->masks[i]); 9184bc63b1bSTonghao Zhang 9194bc63b1bSTonghao Zhang if (t && mask_equal(mask, t)) 9204bc63b1bSTonghao Zhang return t; 921e6445719SPravin B Shelar } 922e6445719SPravin B Shelar 923e6445719SPravin B Shelar return NULL; 924e6445719SPravin B Shelar } 925e6445719SPravin B Shelar 926d1211908SBen Pfaff /* Add 'mask' into the mask list, if it is not already there. */ 927618ed0c8SPravin B Shelar static int flow_mask_insert(struct flow_table *tbl, struct sw_flow *flow, 92812eb18f7SThomas Graf const struct sw_flow_mask *new) 929e6445719SPravin B Shelar { 930618ed0c8SPravin B Shelar struct sw_flow_mask *mask; 9314bc63b1bSTonghao Zhang 932618ed0c8SPravin B Shelar mask = flow_mask_find(tbl, new); 933618ed0c8SPravin B Shelar if (!mask) { 934618ed0c8SPravin B Shelar /* Allocate a new mask if none exsits. */ 935618ed0c8SPravin B Shelar mask = mask_alloc(); 936618ed0c8SPravin B Shelar if (!mask) 937618ed0c8SPravin B Shelar return -ENOMEM; 938618ed0c8SPravin B Shelar mask->key = new->key; 939618ed0c8SPravin B Shelar mask->range = new->range; 9404bc63b1bSTonghao Zhang 9414bc63b1bSTonghao Zhang /* Add mask to mask-list. */ 94257f7d7b9STonghao Zhang if (tbl_mask_array_add_mask(tbl, mask)) { 9434bc63b1bSTonghao Zhang kfree(mask); 94457f7d7b9STonghao Zhang return -ENOMEM; 9454bc63b1bSTonghao Zhang } 946e80857ccSAndy Zhou } else { 947e80857ccSAndy Zhou BUG_ON(!mask->ref_count); 948e80857ccSAndy Zhou mask->ref_count++; 949e6445719SPravin B Shelar } 950e6445719SPravin B Shelar 951618ed0c8SPravin B Shelar flow->mask = mask; 952618ed0c8SPravin B Shelar return 0; 953618ed0c8SPravin B Shelar } 954618ed0c8SPravin B Shelar 95556c19868SJarno Rajahalme /* Must be called with OVS mutex held. */ 956d29ab6f8SJoe Stringer static void flow_key_insert(struct flow_table *table, struct sw_flow *flow) 957618ed0c8SPravin B Shelar { 958618ed0c8SPravin B Shelar struct table_instance *new_ti = NULL; 959618ed0c8SPravin B Shelar struct table_instance *ti; 960618ed0c8SPravin B Shelar 96174ed7ab9SJoe Stringer flow->flow_table.hash = flow_hash(&flow->key, &flow->mask->range); 962618ed0c8SPravin B Shelar ti = ovsl_dereference(table->ti); 963618ed0c8SPravin B Shelar table_instance_insert(ti, flow); 964618ed0c8SPravin B Shelar table->count++; 965618ed0c8SPravin B Shelar 966618ed0c8SPravin B Shelar /* Expand table, if necessary, to make room. */ 967618ed0c8SPravin B Shelar if (table->count > ti->n_buckets) 96874ed7ab9SJoe Stringer new_ti = table_instance_expand(ti, false); 969618ed0c8SPravin B Shelar else if (time_after(jiffies, table->last_rehash + REHASH_INTERVAL)) 97074ed7ab9SJoe Stringer new_ti = table_instance_rehash(ti, ti->n_buckets, false); 971618ed0c8SPravin B Shelar 972618ed0c8SPravin B Shelar if (new_ti) { 973618ed0c8SPravin B Shelar rcu_assign_pointer(table->ti, new_ti); 97474ed7ab9SJoe Stringer call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb); 975618ed0c8SPravin B Shelar table->last_rehash = jiffies; 976618ed0c8SPravin B Shelar } 977d29ab6f8SJoe Stringer } 978d29ab6f8SJoe Stringer 979d29ab6f8SJoe Stringer /* Must be called with OVS mutex held. */ 98074ed7ab9SJoe Stringer static void flow_ufid_insert(struct flow_table *table, struct sw_flow *flow) 98174ed7ab9SJoe Stringer { 98274ed7ab9SJoe Stringer struct table_instance *ti; 98374ed7ab9SJoe Stringer 98474ed7ab9SJoe Stringer flow->ufid_table.hash = ufid_hash(&flow->id); 98574ed7ab9SJoe Stringer ti = ovsl_dereference(table->ufid_ti); 98674ed7ab9SJoe Stringer ufid_table_instance_insert(ti, flow); 98774ed7ab9SJoe Stringer table->ufid_count++; 98874ed7ab9SJoe Stringer 98974ed7ab9SJoe Stringer /* Expand table, if necessary, to make room. */ 99074ed7ab9SJoe Stringer if (table->ufid_count > ti->n_buckets) { 99174ed7ab9SJoe Stringer struct table_instance *new_ti; 99274ed7ab9SJoe Stringer 99374ed7ab9SJoe Stringer new_ti = table_instance_expand(ti, true); 99474ed7ab9SJoe Stringer if (new_ti) { 99574ed7ab9SJoe Stringer rcu_assign_pointer(table->ufid_ti, new_ti); 99674ed7ab9SJoe Stringer call_rcu(&ti->rcu, flow_tbl_destroy_rcu_cb); 99774ed7ab9SJoe Stringer } 99874ed7ab9SJoe Stringer } 99974ed7ab9SJoe Stringer } 100074ed7ab9SJoe Stringer 100174ed7ab9SJoe Stringer /* Must be called with OVS mutex held. */ 1002d29ab6f8SJoe Stringer int ovs_flow_tbl_insert(struct flow_table *table, struct sw_flow *flow, 1003d29ab6f8SJoe Stringer const struct sw_flow_mask *mask) 1004d29ab6f8SJoe Stringer { 1005d29ab6f8SJoe Stringer int err; 1006d29ab6f8SJoe Stringer 1007d29ab6f8SJoe Stringer err = flow_mask_insert(table, flow, mask); 1008d29ab6f8SJoe Stringer if (err) 1009d29ab6f8SJoe Stringer return err; 1010d29ab6f8SJoe Stringer flow_key_insert(table, flow); 101174ed7ab9SJoe Stringer if (ovs_identifier_is_ufid(&flow->id)) 101274ed7ab9SJoe Stringer flow_ufid_insert(table, flow); 1013d29ab6f8SJoe Stringer 1014618ed0c8SPravin B Shelar return 0; 1015618ed0c8SPravin B Shelar } 1016618ed0c8SPravin B Shelar 1017eac87c41SEelco Chaudron static int compare_mask_and_count(const void *a, const void *b) 1018eac87c41SEelco Chaudron { 1019eac87c41SEelco Chaudron const struct mask_count *mc_a = a; 1020eac87c41SEelco Chaudron const struct mask_count *mc_b = b; 1021eac87c41SEelco Chaudron 1022eac87c41SEelco Chaudron return (s64)mc_b->counter - (s64)mc_a->counter; 1023eac87c41SEelco Chaudron } 1024eac87c41SEelco Chaudron 1025eac87c41SEelco Chaudron /* Must be called with OVS mutex held. */ 1026eac87c41SEelco Chaudron void ovs_flow_masks_rebalance(struct flow_table *table) 1027eac87c41SEelco Chaudron { 1028eac87c41SEelco Chaudron struct mask_array *ma = rcu_dereference_ovsl(table->mask_array); 1029eac87c41SEelco Chaudron struct mask_count *masks_and_count; 1030eac87c41SEelco Chaudron struct mask_array *new; 1031eac87c41SEelco Chaudron int masks_entries = 0; 1032eac87c41SEelco Chaudron int i; 1033eac87c41SEelco Chaudron 1034eac87c41SEelco Chaudron /* Build array of all current entries with use counters. */ 1035eac87c41SEelco Chaudron masks_and_count = kmalloc_array(ma->max, sizeof(*masks_and_count), 1036eac87c41SEelco Chaudron GFP_KERNEL); 1037eac87c41SEelco Chaudron if (!masks_and_count) 1038eac87c41SEelco Chaudron return; 1039eac87c41SEelco Chaudron 1040eac87c41SEelco Chaudron for (i = 0; i < ma->max; i++) { 1041eac87c41SEelco Chaudron struct sw_flow_mask *mask; 1042eac87c41SEelco Chaudron unsigned int start; 1043eac87c41SEelco Chaudron int cpu; 1044eac87c41SEelco Chaudron 1045eac87c41SEelco Chaudron mask = rcu_dereference_ovsl(ma->masks[i]); 1046eac87c41SEelco Chaudron if (unlikely(!mask)) 1047eac87c41SEelco Chaudron break; 1048eac87c41SEelco Chaudron 1049eac87c41SEelco Chaudron masks_and_count[i].index = i; 1050eac87c41SEelco Chaudron masks_and_count[i].counter = 0; 1051eac87c41SEelco Chaudron 1052eac87c41SEelco Chaudron for_each_possible_cpu(cpu) { 1053eac87c41SEelco Chaudron u64 *usage_counters = per_cpu_ptr(ma->masks_usage_cntr, 1054eac87c41SEelco Chaudron cpu); 1055eac87c41SEelco Chaudron u64 counter; 1056eac87c41SEelco Chaudron 1057eac87c41SEelco Chaudron do { 1058eac87c41SEelco Chaudron start = u64_stats_fetch_begin_irq(&ma->syncp); 1059eac87c41SEelco Chaudron counter = usage_counters[i]; 1060eac87c41SEelco Chaudron } while (u64_stats_fetch_retry_irq(&ma->syncp, start)); 1061eac87c41SEelco Chaudron 1062eac87c41SEelco Chaudron masks_and_count[i].counter += counter; 1063eac87c41SEelco Chaudron } 1064eac87c41SEelco Chaudron 1065eac87c41SEelco Chaudron /* Subtract the zero count value. */ 1066eac87c41SEelco Chaudron masks_and_count[i].counter -= ma->masks_usage_zero_cntr[i]; 1067eac87c41SEelco Chaudron 1068eac87c41SEelco Chaudron /* Rather than calling tbl_mask_array_reset_counters() 1069eac87c41SEelco Chaudron * below when no change is needed, do it inline here. 1070eac87c41SEelco Chaudron */ 1071eac87c41SEelco Chaudron ma->masks_usage_zero_cntr[i] += masks_and_count[i].counter; 1072eac87c41SEelco Chaudron } 1073eac87c41SEelco Chaudron 1074eac87c41SEelco Chaudron if (i == 0) 1075eac87c41SEelco Chaudron goto free_mask_entries; 1076eac87c41SEelco Chaudron 1077eac87c41SEelco Chaudron /* Sort the entries */ 1078eac87c41SEelco Chaudron masks_entries = i; 1079eac87c41SEelco Chaudron sort(masks_and_count, masks_entries, sizeof(*masks_and_count), 1080eac87c41SEelco Chaudron compare_mask_and_count, NULL); 1081eac87c41SEelco Chaudron 1082eac87c41SEelco Chaudron /* If the order is the same, nothing to do... */ 1083eac87c41SEelco Chaudron for (i = 0; i < masks_entries; i++) { 1084eac87c41SEelco Chaudron if (i != masks_and_count[i].index) 1085eac87c41SEelco Chaudron break; 1086eac87c41SEelco Chaudron } 1087eac87c41SEelco Chaudron if (i == masks_entries) 1088eac87c41SEelco Chaudron goto free_mask_entries; 1089eac87c41SEelco Chaudron 1090eac87c41SEelco Chaudron /* Rebuilt the new list in order of usage. */ 1091eac87c41SEelco Chaudron new = tbl_mask_array_alloc(ma->max); 1092eac87c41SEelco Chaudron if (!new) 1093eac87c41SEelco Chaudron goto free_mask_entries; 1094eac87c41SEelco Chaudron 1095eac87c41SEelco Chaudron for (i = 0; i < masks_entries; i++) { 1096eac87c41SEelco Chaudron int index = masks_and_count[i].index; 1097eac87c41SEelco Chaudron 1098eac87c41SEelco Chaudron new->masks[new->count++] = 1099eac87c41SEelco Chaudron rcu_dereference_ovsl(ma->masks[index]); 1100eac87c41SEelco Chaudron } 1101eac87c41SEelco Chaudron 1102eac87c41SEelco Chaudron rcu_assign_pointer(table->mask_array, new); 1103eac87c41SEelco Chaudron call_rcu(&ma->rcu, mask_array_rcu_cb); 1104eac87c41SEelco Chaudron 1105eac87c41SEelco Chaudron free_mask_entries: 1106eac87c41SEelco Chaudron kfree(masks_and_count); 1107eac87c41SEelco Chaudron } 1108eac87c41SEelco Chaudron 1109e6445719SPravin B Shelar /* Initializes the flow module. 1110e6445719SPravin B Shelar * Returns zero if successful or a negative error code. */ 1111e6445719SPravin B Shelar int ovs_flow_init(void) 1112e6445719SPravin B Shelar { 1113e6445719SPravin B Shelar BUILD_BUG_ON(__alignof__(struct sw_flow_key) % __alignof__(long)); 1114e6445719SPravin B Shelar BUILD_BUG_ON(sizeof(struct sw_flow_key) % sizeof(long)); 1115e6445719SPravin B Shelar 111663e7959cSJarno Rajahalme flow_cache = kmem_cache_create("sw_flow", sizeof(struct sw_flow) 1117db74a333SThadeu Lima de Souza Cascardo + (nr_cpu_ids 1118aef833c5SPablo Neira Ayuso * sizeof(struct sw_flow_stats *)), 111963e7959cSJarno Rajahalme 0, 0, NULL); 1120e6445719SPravin B Shelar if (flow_cache == NULL) 1121e6445719SPravin B Shelar return -ENOMEM; 1122e6445719SPravin B Shelar 112363e7959cSJarno Rajahalme flow_stats_cache 1124aef833c5SPablo Neira Ayuso = kmem_cache_create("sw_flow_stats", sizeof(struct sw_flow_stats), 112563e7959cSJarno Rajahalme 0, SLAB_HWCACHE_ALIGN, NULL); 112663e7959cSJarno Rajahalme if (flow_stats_cache == NULL) { 112763e7959cSJarno Rajahalme kmem_cache_destroy(flow_cache); 112863e7959cSJarno Rajahalme flow_cache = NULL; 112963e7959cSJarno Rajahalme return -ENOMEM; 113063e7959cSJarno Rajahalme } 113163e7959cSJarno Rajahalme 1132e6445719SPravin B Shelar return 0; 1133e6445719SPravin B Shelar } 1134e6445719SPravin B Shelar 1135e6445719SPravin B Shelar /* Uninitializes the flow module. */ 1136e6445719SPravin B Shelar void ovs_flow_exit(void) 1137e6445719SPravin B Shelar { 113863e7959cSJarno Rajahalme kmem_cache_destroy(flow_stats_cache); 1139e6445719SPravin B Shelar kmem_cache_destroy(flow_cache); 1140e6445719SPravin B Shelar } 1141