1 /* 2 * Copyright (c) 2016 Citrix Systems Inc. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License version 2 6 * as published by the Free Softare Foundation; or, when distributed 7 * separately from the Linux kernel or incorporated into other 8 * software packages, subject to the following license: 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a copy 11 * of this source file (the "Software"), to deal in the Software without 12 * restriction, including without limitation the rights to use, copy, modify, 13 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 14 * and to permit persons to whom the Software is furnished to do so, subject to 15 * the following conditions: 16 * 17 * The above copyright notice and this permission notice shall be included in 18 * all copies or substantial portions of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 26 * IN THE SOFTWARE. 27 */ 28 29 #define XEN_NETIF_DEFINE_TOEPLITZ 30 31 #include "common.h" 32 #include <linux/vmalloc.h> 33 #include <linux/rculist.h> 34 35 static void xenvif_del_hash(struct rcu_head *rcu) 36 { 37 struct xenvif_hash_cache_entry *entry; 38 39 entry = container_of(rcu, struct xenvif_hash_cache_entry, rcu); 40 41 kfree(entry); 42 } 43 44 static void xenvif_add_hash(struct xenvif *vif, const u8 *tag, 45 unsigned int len, u32 val) 46 { 47 struct xenvif_hash_cache_entry *new, *entry, *oldest; 48 unsigned long flags; 49 bool found; 50 51 new = kmalloc(sizeof(*entry), GFP_KERNEL); 52 if (!new) 53 return; 54 55 memcpy(new->tag, tag, len); 56 new->len = len; 57 new->val = val; 58 59 spin_lock_irqsave(&vif->hash.cache.lock, flags); 60 61 found = false; 62 oldest = NULL; 63 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 64 /* Make sure we don't add duplicate entries */ 65 if (entry->len == len && 66 memcmp(entry->tag, tag, len) == 0) 67 found = true; 68 if (!oldest || entry->seq < oldest->seq) 69 oldest = entry; 70 } 71 72 if (!found) { 73 new->seq = atomic_inc_return(&vif->hash.cache.seq); 74 list_add_rcu(&new->link, &vif->hash.cache.list); 75 76 if (++vif->hash.cache.count > xenvif_hash_cache_size) { 77 list_del_rcu(&oldest->link); 78 vif->hash.cache.count--; 79 call_rcu(&oldest->rcu, xenvif_del_hash); 80 } 81 } 82 83 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 84 85 if (found) 86 kfree(new); 87 } 88 89 static u32 xenvif_new_hash(struct xenvif *vif, const u8 *data, 90 unsigned int len) 91 { 92 u32 val; 93 94 val = xen_netif_toeplitz_hash(vif->hash.key, 95 sizeof(vif->hash.key), 96 data, len); 97 98 if (xenvif_hash_cache_size != 0) 99 xenvif_add_hash(vif, data, len, val); 100 101 return val; 102 } 103 104 static void xenvif_flush_hash(struct xenvif *vif) 105 { 106 struct xenvif_hash_cache_entry *entry; 107 unsigned long flags; 108 109 if (xenvif_hash_cache_size == 0) 110 return; 111 112 spin_lock_irqsave(&vif->hash.cache.lock, flags); 113 114 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 115 list_del_rcu(&entry->link); 116 vif->hash.cache.count--; 117 call_rcu(&entry->rcu, xenvif_del_hash); 118 } 119 120 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 121 } 122 123 static u32 xenvif_find_hash(struct xenvif *vif, const u8 *data, 124 unsigned int len) 125 { 126 struct xenvif_hash_cache_entry *entry; 127 u32 val; 128 bool found; 129 130 if (len >= XEN_NETBK_HASH_TAG_SIZE) 131 return 0; 132 133 if (xenvif_hash_cache_size == 0) 134 return xenvif_new_hash(vif, data, len); 135 136 rcu_read_lock(); 137 138 found = false; 139 140 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 141 if (entry->len == len && 142 memcmp(entry->tag, data, len) == 0) { 143 val = entry->val; 144 entry->seq = atomic_inc_return(&vif->hash.cache.seq); 145 found = true; 146 break; 147 } 148 } 149 150 rcu_read_unlock(); 151 152 if (!found) 153 val = xenvif_new_hash(vif, data, len); 154 155 return val; 156 } 157 158 void xenvif_set_skb_hash(struct xenvif *vif, struct sk_buff *skb) 159 { 160 struct flow_keys flow; 161 u32 hash = 0; 162 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 163 u32 flags = vif->hash.flags; 164 bool has_tcp_hdr; 165 166 /* Quick rejection test: If the network protocol doesn't 167 * correspond to any enabled hash type then there's no point 168 * in parsing the packet header. 169 */ 170 switch (skb->protocol) { 171 case htons(ETH_P_IP): 172 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 173 XEN_NETIF_CTRL_HASH_TYPE_IPV4)) 174 break; 175 176 goto done; 177 178 case htons(ETH_P_IPV6): 179 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP | 180 XEN_NETIF_CTRL_HASH_TYPE_IPV6)) 181 break; 182 183 goto done; 184 185 default: 186 goto done; 187 } 188 189 memset(&flow, 0, sizeof(flow)); 190 if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) 191 goto done; 192 193 has_tcp_hdr = (flow.basic.ip_proto == IPPROTO_TCP) && 194 !(flow.control.flags & FLOW_DIS_IS_FRAGMENT); 195 196 switch (skb->protocol) { 197 case htons(ETH_P_IP): 198 if (has_tcp_hdr && 199 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP)) { 200 u8 data[12]; 201 202 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 203 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 204 memcpy(&data[8], &flow.ports.src, 2); 205 memcpy(&data[10], &flow.ports.dst, 2); 206 207 hash = xenvif_find_hash(vif, data, sizeof(data)); 208 type = PKT_HASH_TYPE_L4; 209 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) { 210 u8 data[8]; 211 212 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 213 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 214 215 hash = xenvif_find_hash(vif, data, sizeof(data)); 216 type = PKT_HASH_TYPE_L3; 217 } 218 219 break; 220 221 case htons(ETH_P_IPV6): 222 if (has_tcp_hdr && 223 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) { 224 u8 data[36]; 225 226 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 227 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 228 memcpy(&data[32], &flow.ports.src, 2); 229 memcpy(&data[34], &flow.ports.dst, 2); 230 231 hash = xenvif_find_hash(vif, data, sizeof(data)); 232 type = PKT_HASH_TYPE_L4; 233 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) { 234 u8 data[32]; 235 236 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 237 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 238 239 hash = xenvif_find_hash(vif, data, sizeof(data)); 240 type = PKT_HASH_TYPE_L3; 241 } 242 243 break; 244 } 245 246 done: 247 if (type == PKT_HASH_TYPE_NONE) 248 skb_clear_hash(skb); 249 else 250 __skb_set_sw_hash(skb, hash, type == PKT_HASH_TYPE_L4); 251 } 252 253 u32 xenvif_set_hash_alg(struct xenvif *vif, u32 alg) 254 { 255 switch (alg) { 256 case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE: 257 case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ: 258 break; 259 260 default: 261 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 262 } 263 264 vif->hash.alg = alg; 265 266 return XEN_NETIF_CTRL_STATUS_SUCCESS; 267 } 268 269 u32 xenvif_get_hash_flags(struct xenvif *vif, u32 *flags) 270 { 271 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 272 return XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 273 274 *flags = XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 275 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 276 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 277 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP; 278 279 return XEN_NETIF_CTRL_STATUS_SUCCESS; 280 } 281 282 u32 xenvif_set_hash_flags(struct xenvif *vif, u32 flags) 283 { 284 if (flags & ~(XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 285 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 286 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 287 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) 288 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 289 290 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 291 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 292 293 vif->hash.flags = flags; 294 295 return XEN_NETIF_CTRL_STATUS_SUCCESS; 296 } 297 298 u32 xenvif_set_hash_key(struct xenvif *vif, u32 gref, u32 len) 299 { 300 u8 *key = vif->hash.key; 301 struct gnttab_copy copy_op = { 302 .source.u.ref = gref, 303 .source.domid = vif->domid, 304 .dest.u.gmfn = virt_to_gfn(key), 305 .dest.domid = DOMID_SELF, 306 .dest.offset = xen_offset_in_page(key), 307 .len = len, 308 .flags = GNTCOPY_source_gref 309 }; 310 311 if (len > XEN_NETBK_MAX_HASH_KEY_SIZE) 312 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 313 314 if (copy_op.len != 0) { 315 gnttab_batch_copy(©_op, 1); 316 317 if (copy_op.status != GNTST_okay) 318 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 319 } 320 321 /* Clear any remaining key octets */ 322 if (len < XEN_NETBK_MAX_HASH_KEY_SIZE) 323 memset(key + len, 0, XEN_NETBK_MAX_HASH_KEY_SIZE - len); 324 325 xenvif_flush_hash(vif); 326 327 return XEN_NETIF_CTRL_STATUS_SUCCESS; 328 } 329 330 u32 xenvif_set_hash_mapping_size(struct xenvif *vif, u32 size) 331 { 332 if (size > XEN_NETBK_MAX_HASH_MAPPING_SIZE) 333 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 334 335 vif->hash.size = size; 336 memset(vif->hash.mapping, 0, sizeof(u32) * size); 337 338 return XEN_NETIF_CTRL_STATUS_SUCCESS; 339 } 340 341 u32 xenvif_set_hash_mapping(struct xenvif *vif, u32 gref, u32 len, 342 u32 off) 343 { 344 u32 *mapping = &vif->hash.mapping[off]; 345 struct gnttab_copy copy_op = { 346 .source.u.ref = gref, 347 .source.domid = vif->domid, 348 .dest.u.gmfn = virt_to_gfn(mapping), 349 .dest.domid = DOMID_SELF, 350 .dest.offset = xen_offset_in_page(mapping), 351 .len = len * sizeof(u32), 352 .flags = GNTCOPY_source_gref 353 }; 354 355 if ((off + len > vif->hash.size) || copy_op.len > XEN_PAGE_SIZE) 356 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 357 358 while (len-- != 0) 359 if (mapping[off++] >= vif->num_queues) 360 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 361 362 if (copy_op.len != 0) { 363 gnttab_batch_copy(©_op, 1); 364 365 if (copy_op.status != GNTST_okay) 366 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 367 } 368 369 return XEN_NETIF_CTRL_STATUS_SUCCESS; 370 } 371 372 void xenvif_init_hash(struct xenvif *vif) 373 { 374 if (xenvif_hash_cache_size == 0) 375 return; 376 377 spin_lock_init(&vif->hash.cache.lock); 378 INIT_LIST_HEAD(&vif->hash.cache.list); 379 } 380 381 void xenvif_deinit_hash(struct xenvif *vif) 382 { 383 xenvif_flush_hash(vif); 384 } 385