1 /* 2 * Copyright (c) 2016 Citrix Systems Inc. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License version 2 6 * as published by the Free Softare Foundation; or, when distributed 7 * separately from the Linux kernel or incorporated into other 8 * software packages, subject to the following license: 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a copy 11 * of this source file (the "Software"), to deal in the Software without 12 * restriction, including without limitation the rights to use, copy, modify, 13 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 14 * and to permit persons to whom the Software is furnished to do so, subject to 15 * the following conditions: 16 * 17 * The above copyright notice and this permission notice shall be included in 18 * all copies or substantial portions of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 26 * IN THE SOFTWARE. 27 */ 28 29 #define XEN_NETIF_DEFINE_TOEPLITZ 30 31 #include "common.h" 32 #include <linux/vmalloc.h> 33 #include <linux/rculist.h> 34 35 static void xenvif_add_hash(struct xenvif *vif, const u8 *tag, 36 unsigned int len, u32 val) 37 { 38 struct xenvif_hash_cache_entry *new, *entry, *oldest; 39 unsigned long flags; 40 bool found; 41 42 new = kmalloc(sizeof(*entry), GFP_ATOMIC); 43 if (!new) 44 return; 45 46 memcpy(new->tag, tag, len); 47 new->len = len; 48 new->val = val; 49 50 spin_lock_irqsave(&vif->hash.cache.lock, flags); 51 52 found = false; 53 oldest = NULL; 54 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 55 /* Make sure we don't add duplicate entries */ 56 if (entry->len == len && 57 memcmp(entry->tag, tag, len) == 0) 58 found = true; 59 if (!oldest || entry->seq < oldest->seq) 60 oldest = entry; 61 } 62 63 if (!found) { 64 new->seq = atomic_inc_return(&vif->hash.cache.seq); 65 list_add_rcu(&new->link, &vif->hash.cache.list); 66 67 if (++vif->hash.cache.count > xenvif_hash_cache_size) { 68 list_del_rcu(&oldest->link); 69 vif->hash.cache.count--; 70 kfree_rcu(oldest, rcu); 71 } 72 } 73 74 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 75 76 if (found) 77 kfree(new); 78 } 79 80 static u32 xenvif_new_hash(struct xenvif *vif, const u8 *data, 81 unsigned int len) 82 { 83 u32 val; 84 85 val = xen_netif_toeplitz_hash(vif->hash.key, 86 sizeof(vif->hash.key), 87 data, len); 88 89 if (xenvif_hash_cache_size != 0) 90 xenvif_add_hash(vif, data, len, val); 91 92 return val; 93 } 94 95 static void xenvif_flush_hash(struct xenvif *vif) 96 { 97 struct xenvif_hash_cache_entry *entry; 98 unsigned long flags; 99 100 if (xenvif_hash_cache_size == 0) 101 return; 102 103 spin_lock_irqsave(&vif->hash.cache.lock, flags); 104 105 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 106 list_del_rcu(&entry->link); 107 vif->hash.cache.count--; 108 kfree_rcu(entry, rcu); 109 } 110 111 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 112 } 113 114 static u32 xenvif_find_hash(struct xenvif *vif, const u8 *data, 115 unsigned int len) 116 { 117 struct xenvif_hash_cache_entry *entry; 118 u32 val; 119 bool found; 120 121 if (len >= XEN_NETBK_HASH_TAG_SIZE) 122 return 0; 123 124 if (xenvif_hash_cache_size == 0) 125 return xenvif_new_hash(vif, data, len); 126 127 rcu_read_lock(); 128 129 found = false; 130 131 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 132 if (entry->len == len && 133 memcmp(entry->tag, data, len) == 0) { 134 val = entry->val; 135 entry->seq = atomic_inc_return(&vif->hash.cache.seq); 136 found = true; 137 break; 138 } 139 } 140 141 rcu_read_unlock(); 142 143 if (!found) 144 val = xenvif_new_hash(vif, data, len); 145 146 return val; 147 } 148 149 void xenvif_set_skb_hash(struct xenvif *vif, struct sk_buff *skb) 150 { 151 struct flow_keys flow; 152 u32 hash = 0; 153 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 154 u32 flags = vif->hash.flags; 155 bool has_tcp_hdr; 156 157 /* Quick rejection test: If the network protocol doesn't 158 * correspond to any enabled hash type then there's no point 159 * in parsing the packet header. 160 */ 161 switch (skb->protocol) { 162 case htons(ETH_P_IP): 163 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 164 XEN_NETIF_CTRL_HASH_TYPE_IPV4)) 165 break; 166 167 goto done; 168 169 case htons(ETH_P_IPV6): 170 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP | 171 XEN_NETIF_CTRL_HASH_TYPE_IPV6)) 172 break; 173 174 goto done; 175 176 default: 177 goto done; 178 } 179 180 memset(&flow, 0, sizeof(flow)); 181 if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) 182 goto done; 183 184 has_tcp_hdr = (flow.basic.ip_proto == IPPROTO_TCP) && 185 !(flow.control.flags & FLOW_DIS_IS_FRAGMENT); 186 187 switch (skb->protocol) { 188 case htons(ETH_P_IP): 189 if (has_tcp_hdr && 190 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP)) { 191 u8 data[12]; 192 193 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 194 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 195 memcpy(&data[8], &flow.ports.src, 2); 196 memcpy(&data[10], &flow.ports.dst, 2); 197 198 hash = xenvif_find_hash(vif, data, sizeof(data)); 199 type = PKT_HASH_TYPE_L4; 200 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) { 201 u8 data[8]; 202 203 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 204 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 205 206 hash = xenvif_find_hash(vif, data, sizeof(data)); 207 type = PKT_HASH_TYPE_L3; 208 } 209 210 break; 211 212 case htons(ETH_P_IPV6): 213 if (has_tcp_hdr && 214 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) { 215 u8 data[36]; 216 217 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 218 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 219 memcpy(&data[32], &flow.ports.src, 2); 220 memcpy(&data[34], &flow.ports.dst, 2); 221 222 hash = xenvif_find_hash(vif, data, sizeof(data)); 223 type = PKT_HASH_TYPE_L4; 224 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) { 225 u8 data[32]; 226 227 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 228 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 229 230 hash = xenvif_find_hash(vif, data, sizeof(data)); 231 type = PKT_HASH_TYPE_L3; 232 } 233 234 break; 235 } 236 237 done: 238 if (type == PKT_HASH_TYPE_NONE) 239 skb_clear_hash(skb); 240 else 241 __skb_set_sw_hash(skb, hash, type == PKT_HASH_TYPE_L4); 242 } 243 244 u32 xenvif_set_hash_alg(struct xenvif *vif, u32 alg) 245 { 246 switch (alg) { 247 case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE: 248 case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ: 249 break; 250 251 default: 252 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 253 } 254 255 vif->hash.alg = alg; 256 257 return XEN_NETIF_CTRL_STATUS_SUCCESS; 258 } 259 260 u32 xenvif_get_hash_flags(struct xenvif *vif, u32 *flags) 261 { 262 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 263 return XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 264 265 *flags = XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 266 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 267 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 268 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP; 269 270 return XEN_NETIF_CTRL_STATUS_SUCCESS; 271 } 272 273 u32 xenvif_set_hash_flags(struct xenvif *vif, u32 flags) 274 { 275 if (flags & ~(XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 276 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 277 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 278 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) 279 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 280 281 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 282 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 283 284 vif->hash.flags = flags; 285 286 return XEN_NETIF_CTRL_STATUS_SUCCESS; 287 } 288 289 u32 xenvif_set_hash_key(struct xenvif *vif, u32 gref, u32 len) 290 { 291 u8 *key = vif->hash.key; 292 struct gnttab_copy copy_op = { 293 .source.u.ref = gref, 294 .source.domid = vif->domid, 295 .dest.u.gmfn = virt_to_gfn(key), 296 .dest.domid = DOMID_SELF, 297 .dest.offset = xen_offset_in_page(key), 298 .len = len, 299 .flags = GNTCOPY_source_gref 300 }; 301 302 if (len > XEN_NETBK_MAX_HASH_KEY_SIZE) 303 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 304 305 if (copy_op.len != 0) { 306 gnttab_batch_copy(©_op, 1); 307 308 if (copy_op.status != GNTST_okay) 309 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 310 } 311 312 /* Clear any remaining key octets */ 313 if (len < XEN_NETBK_MAX_HASH_KEY_SIZE) 314 memset(key + len, 0, XEN_NETBK_MAX_HASH_KEY_SIZE - len); 315 316 xenvif_flush_hash(vif); 317 318 return XEN_NETIF_CTRL_STATUS_SUCCESS; 319 } 320 321 u32 xenvif_set_hash_mapping_size(struct xenvif *vif, u32 size) 322 { 323 if (size > XEN_NETBK_MAX_HASH_MAPPING_SIZE) 324 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 325 326 vif->hash.size = size; 327 memset(vif->hash.mapping[vif->hash.mapping_sel], 0, 328 sizeof(u32) * size); 329 330 return XEN_NETIF_CTRL_STATUS_SUCCESS; 331 } 332 333 u32 xenvif_set_hash_mapping(struct xenvif *vif, u32 gref, u32 len, 334 u32 off) 335 { 336 u32 *mapping = vif->hash.mapping[!vif->hash.mapping_sel]; 337 unsigned int nr = 1; 338 struct gnttab_copy copy_op[2] = {{ 339 .source.u.ref = gref, 340 .source.domid = vif->domid, 341 .dest.domid = DOMID_SELF, 342 .len = len * sizeof(*mapping), 343 .flags = GNTCOPY_source_gref 344 }}; 345 346 if ((off + len < off) || (off + len > vif->hash.size) || 347 len > XEN_PAGE_SIZE / sizeof(*mapping)) 348 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 349 350 copy_op[0].dest.u.gmfn = virt_to_gfn(mapping + off); 351 copy_op[0].dest.offset = xen_offset_in_page(mapping + off); 352 if (copy_op[0].dest.offset + copy_op[0].len > XEN_PAGE_SIZE) { 353 copy_op[1] = copy_op[0]; 354 copy_op[1].source.offset = XEN_PAGE_SIZE - copy_op[0].dest.offset; 355 copy_op[1].dest.u.gmfn = virt_to_gfn(mapping + off + len); 356 copy_op[1].dest.offset = 0; 357 copy_op[1].len = copy_op[0].len - copy_op[1].source.offset; 358 copy_op[0].len = copy_op[1].source.offset; 359 nr = 2; 360 } 361 362 memcpy(mapping, vif->hash.mapping[vif->hash.mapping_sel], 363 vif->hash.size * sizeof(*mapping)); 364 365 if (copy_op[0].len != 0) { 366 gnttab_batch_copy(copy_op, nr); 367 368 if (copy_op[0].status != GNTST_okay || 369 copy_op[nr - 1].status != GNTST_okay) 370 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 371 } 372 373 while (len-- != 0) 374 if (mapping[off++] >= vif->num_queues) 375 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 376 377 vif->hash.mapping_sel = !vif->hash.mapping_sel; 378 379 return XEN_NETIF_CTRL_STATUS_SUCCESS; 380 } 381 382 #ifdef CONFIG_DEBUG_FS 383 void xenvif_dump_hash_info(struct xenvif *vif, struct seq_file *m) 384 { 385 unsigned int i; 386 387 switch (vif->hash.alg) { 388 case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ: 389 seq_puts(m, "Hash Algorithm: TOEPLITZ\n"); 390 break; 391 392 case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE: 393 seq_puts(m, "Hash Algorithm: NONE\n"); 394 /* FALLTHRU */ 395 default: 396 return; 397 } 398 399 if (vif->hash.flags) { 400 seq_puts(m, "\nHash Flags:\n"); 401 402 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) 403 seq_puts(m, "- IPv4\n"); 404 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP) 405 seq_puts(m, "- IPv4 + TCP\n"); 406 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) 407 seq_puts(m, "- IPv6\n"); 408 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP) 409 seq_puts(m, "- IPv6 + TCP\n"); 410 } 411 412 seq_puts(m, "\nHash Key:\n"); 413 414 for (i = 0; i < XEN_NETBK_MAX_HASH_KEY_SIZE; ) { 415 unsigned int j, n; 416 417 n = 8; 418 if (i + n >= XEN_NETBK_MAX_HASH_KEY_SIZE) 419 n = XEN_NETBK_MAX_HASH_KEY_SIZE - i; 420 421 seq_printf(m, "[%2u - %2u]: ", i, i + n - 1); 422 423 for (j = 0; j < n; j++, i++) 424 seq_printf(m, "%02x ", vif->hash.key[i]); 425 426 seq_puts(m, "\n"); 427 } 428 429 if (vif->hash.size != 0) { 430 const u32 *mapping = vif->hash.mapping[vif->hash.mapping_sel]; 431 432 seq_puts(m, "\nHash Mapping:\n"); 433 434 for (i = 0; i < vif->hash.size; ) { 435 unsigned int j, n; 436 437 n = 8; 438 if (i + n >= vif->hash.size) 439 n = vif->hash.size - i; 440 441 seq_printf(m, "[%4u - %4u]: ", i, i + n - 1); 442 443 for (j = 0; j < n; j++, i++) 444 seq_printf(m, "%4u ", mapping[i]); 445 446 seq_puts(m, "\n"); 447 } 448 } 449 } 450 #endif /* CONFIG_DEBUG_FS */ 451 452 void xenvif_init_hash(struct xenvif *vif) 453 { 454 if (xenvif_hash_cache_size == 0) 455 return; 456 457 BUG_ON(vif->hash.cache.count); 458 459 spin_lock_init(&vif->hash.cache.lock); 460 INIT_LIST_HEAD(&vif->hash.cache.list); 461 } 462 463 void xenvif_deinit_hash(struct xenvif *vif) 464 { 465 xenvif_flush_hash(vif); 466 } 467