1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2005-2018 Solarflare Communications Inc. 5 * Copyright 2019-2020 Xilinx Inc. 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 as published 9 * by the Free Software Foundation, incorporated herein by reference. 10 */ 11 12 #include "mcdi_filters.h" 13 #include "mcdi.h" 14 #include "nic.h" 15 #include "rx_common.h" 16 17 /* The maximum size of a shared RSS context */ 18 /* TODO: this should really be from the mcdi protocol export */ 19 #define EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE 64UL 20 21 #define EFX_EF10_FILTER_ID_INVALID 0xffff 22 23 /* An arbitrary search limit for the software hash table */ 24 #define EFX_EF10_FILTER_SEARCH_LIMIT 200 25 26 static struct efx_filter_spec * 27 efx_mcdi_filter_entry_spec(const struct efx_mcdi_filter_table *table, 28 unsigned int filter_idx) 29 { 30 return (struct efx_filter_spec *)(table->entry[filter_idx].spec & 31 ~EFX_EF10_FILTER_FLAGS); 32 } 33 34 static unsigned int 35 efx_mcdi_filter_entry_flags(const struct efx_mcdi_filter_table *table, 36 unsigned int filter_idx) 37 { 38 return table->entry[filter_idx].spec & EFX_EF10_FILTER_FLAGS; 39 } 40 41 static u32 efx_mcdi_filter_get_unsafe_id(u32 filter_id) 42 { 43 WARN_ON_ONCE(filter_id == EFX_EF10_FILTER_ID_INVALID); 44 return filter_id & (EFX_MCDI_FILTER_TBL_ROWS - 1); 45 } 46 47 static unsigned int efx_mcdi_filter_get_unsafe_pri(u32 filter_id) 48 { 49 return filter_id / (EFX_MCDI_FILTER_TBL_ROWS * 2); 50 } 51 52 static u32 efx_mcdi_filter_make_filter_id(unsigned int pri, u16 idx) 53 { 54 return pri * EFX_MCDI_FILTER_TBL_ROWS * 2 + idx; 55 } 56 57 /* 58 * Decide whether a filter should be exclusive or else should allow 59 * delivery to additional recipients. Currently we decide that 60 * filters for specific local unicast MAC and IP addresses are 61 * exclusive. 62 */ 63 static bool efx_mcdi_filter_is_exclusive(const struct efx_filter_spec *spec) 64 { 65 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC && 66 !is_multicast_ether_addr(spec->loc_mac)) 67 return true; 68 69 if ((spec->match_flags & 70 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) == 71 (EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_LOC_HOST)) { 72 if (spec->ether_type == htons(ETH_P_IP) && 73 !ipv4_is_multicast(spec->loc_host[0])) 74 return true; 75 if (spec->ether_type == htons(ETH_P_IPV6) && 76 ((const u8 *)spec->loc_host)[0] != 0xff) 77 return true; 78 } 79 80 return false; 81 } 82 83 static void 84 efx_mcdi_filter_set_entry(struct efx_mcdi_filter_table *table, 85 unsigned int filter_idx, 86 const struct efx_filter_spec *spec, 87 unsigned int flags) 88 { 89 table->entry[filter_idx].spec = (unsigned long)spec | flags; 90 } 91 92 static void 93 efx_mcdi_filter_push_prep_set_match_fields(struct efx_nic *efx, 94 const struct efx_filter_spec *spec, 95 efx_dword_t *inbuf) 96 { 97 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 98 u32 match_fields = 0, uc_match, mc_match; 99 100 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 101 efx_mcdi_filter_is_exclusive(spec) ? 102 MC_CMD_FILTER_OP_IN_OP_INSERT : 103 MC_CMD_FILTER_OP_IN_OP_SUBSCRIBE); 104 105 /* 106 * Convert match flags and values. Unlike almost 107 * everything else in MCDI, these fields are in 108 * network byte order. 109 */ 110 #define COPY_VALUE(value, mcdi_field) \ 111 do { \ 112 match_fields |= \ 113 1 << MC_CMD_FILTER_OP_IN_MATCH_ ## \ 114 mcdi_field ## _LBN; \ 115 BUILD_BUG_ON( \ 116 MC_CMD_FILTER_OP_IN_ ## mcdi_field ## _LEN < \ 117 sizeof(value)); \ 118 memcpy(MCDI_PTR(inbuf, FILTER_OP_IN_ ## mcdi_field), \ 119 &value, sizeof(value)); \ 120 } while (0) 121 #define COPY_FIELD(gen_flag, gen_field, mcdi_field) \ 122 if (spec->match_flags & EFX_FILTER_MATCH_ ## gen_flag) { \ 123 COPY_VALUE(spec->gen_field, mcdi_field); \ 124 } 125 /* 126 * Handle encap filters first. They will always be mismatch 127 * (unknown UC or MC) filters 128 */ 129 if (encap_type) { 130 /* 131 * ether_type and outer_ip_proto need to be variables 132 * because COPY_VALUE wants to memcpy them 133 */ 134 __be16 ether_type = 135 htons(encap_type & EFX_ENCAP_FLAG_IPV6 ? 136 ETH_P_IPV6 : ETH_P_IP); 137 u8 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_GENEVE; 138 u8 outer_ip_proto; 139 140 switch (encap_type & EFX_ENCAP_TYPES_MASK) { 141 case EFX_ENCAP_TYPE_VXLAN: 142 vni_type = MC_CMD_FILTER_OP_EXT_IN_VNI_TYPE_VXLAN; 143 fallthrough; 144 case EFX_ENCAP_TYPE_GENEVE: 145 COPY_VALUE(ether_type, ETHER_TYPE); 146 outer_ip_proto = IPPROTO_UDP; 147 COPY_VALUE(outer_ip_proto, IP_PROTO); 148 /* 149 * We always need to set the type field, even 150 * though we're not matching on the TNI. 151 */ 152 MCDI_POPULATE_DWORD_1(inbuf, 153 FILTER_OP_EXT_IN_VNI_OR_VSID, 154 FILTER_OP_EXT_IN_VNI_TYPE, 155 vni_type); 156 break; 157 case EFX_ENCAP_TYPE_NVGRE: 158 COPY_VALUE(ether_type, ETHER_TYPE); 159 outer_ip_proto = IPPROTO_GRE; 160 COPY_VALUE(outer_ip_proto, IP_PROTO); 161 break; 162 default: 163 WARN_ON(1); 164 } 165 166 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 167 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 168 } else { 169 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 170 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 171 } 172 173 if (spec->match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) 174 match_fields |= 175 is_multicast_ether_addr(spec->loc_mac) ? 176 1 << mc_match : 177 1 << uc_match; 178 COPY_FIELD(REM_HOST, rem_host, SRC_IP); 179 COPY_FIELD(LOC_HOST, loc_host, DST_IP); 180 COPY_FIELD(REM_MAC, rem_mac, SRC_MAC); 181 COPY_FIELD(REM_PORT, rem_port, SRC_PORT); 182 COPY_FIELD(LOC_MAC, loc_mac, DST_MAC); 183 COPY_FIELD(LOC_PORT, loc_port, DST_PORT); 184 COPY_FIELD(ETHER_TYPE, ether_type, ETHER_TYPE); 185 COPY_FIELD(INNER_VID, inner_vid, INNER_VLAN); 186 COPY_FIELD(OUTER_VID, outer_vid, OUTER_VLAN); 187 COPY_FIELD(IP_PROTO, ip_proto, IP_PROTO); 188 #undef COPY_FIELD 189 #undef COPY_VALUE 190 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_MATCH_FIELDS, 191 match_fields); 192 } 193 194 static void efx_mcdi_filter_push_prep(struct efx_nic *efx, 195 const struct efx_filter_spec *spec, 196 efx_dword_t *inbuf, u64 handle, 197 struct efx_rss_context *ctx, 198 bool replacing) 199 { 200 u32 flags = spec->flags; 201 202 memset(inbuf, 0, MC_CMD_FILTER_OP_EXT_IN_LEN); 203 204 /* If RSS filter, caller better have given us an RSS context */ 205 if (flags & EFX_FILTER_FLAG_RX_RSS) { 206 /* 207 * We don't have the ability to return an error, so we'll just 208 * log a warning and disable RSS for the filter. 209 */ 210 if (WARN_ON_ONCE(!ctx)) 211 flags &= ~EFX_FILTER_FLAG_RX_RSS; 212 else if (WARN_ON_ONCE(ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID)) 213 flags &= ~EFX_FILTER_FLAG_RX_RSS; 214 } 215 216 if (replacing) { 217 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 218 MC_CMD_FILTER_OP_IN_OP_REPLACE); 219 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, handle); 220 } else { 221 efx_mcdi_filter_push_prep_set_match_fields(efx, spec, inbuf); 222 } 223 224 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_PORT_ID, efx->vport_id); 225 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_DEST, 226 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 227 MC_CMD_FILTER_OP_IN_RX_DEST_DROP : 228 MC_CMD_FILTER_OP_IN_RX_DEST_HOST); 229 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DOMAIN, 0); 230 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_TX_DEST, 231 MC_CMD_FILTER_OP_IN_TX_DEST_DEFAULT); 232 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_QUEUE, 233 spec->dmaq_id == EFX_FILTER_RX_DMAQ_ID_DROP ? 234 0 : spec->dmaq_id); 235 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_MODE, 236 (flags & EFX_FILTER_FLAG_RX_RSS) ? 237 MC_CMD_FILTER_OP_IN_RX_MODE_RSS : 238 MC_CMD_FILTER_OP_IN_RX_MODE_SIMPLE); 239 if (flags & EFX_FILTER_FLAG_RX_RSS) 240 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_RX_CONTEXT, ctx->context_id); 241 } 242 243 static int efx_mcdi_filter_push(struct efx_nic *efx, 244 const struct efx_filter_spec *spec, u64 *handle, 245 struct efx_rss_context *ctx, bool replacing) 246 { 247 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 248 MCDI_DECLARE_BUF(outbuf, MC_CMD_FILTER_OP_EXT_OUT_LEN); 249 size_t outlen; 250 int rc; 251 252 efx_mcdi_filter_push_prep(efx, spec, inbuf, *handle, ctx, replacing); 253 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, sizeof(inbuf), 254 outbuf, sizeof(outbuf), &outlen); 255 if (rc && spec->priority != EFX_FILTER_PRI_HINT) 256 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, sizeof(inbuf), 257 outbuf, outlen, rc); 258 if (rc == 0) 259 *handle = MCDI_QWORD(outbuf, FILTER_OP_OUT_HANDLE); 260 if (rc == -ENOSPC) 261 rc = -EBUSY; /* to match efx_farch_filter_insert() */ 262 return rc; 263 } 264 265 static u32 efx_mcdi_filter_mcdi_flags_from_spec(const struct efx_filter_spec *spec) 266 { 267 enum efx_encap_type encap_type = efx_filter_get_encap_type(spec); 268 unsigned int match_flags = spec->match_flags; 269 unsigned int uc_match, mc_match; 270 u32 mcdi_flags = 0; 271 272 #define MAP_FILTER_TO_MCDI_FLAG(gen_flag, mcdi_field, encap) { \ 273 unsigned int old_match_flags = match_flags; \ 274 match_flags &= ~EFX_FILTER_MATCH_ ## gen_flag; \ 275 if (match_flags != old_match_flags) \ 276 mcdi_flags |= \ 277 (1 << ((encap) ? \ 278 MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_ ## \ 279 mcdi_field ## _LBN : \ 280 MC_CMD_FILTER_OP_EXT_IN_MATCH_ ##\ 281 mcdi_field ## _LBN)); \ 282 } 283 /* inner or outer based on encap type */ 284 MAP_FILTER_TO_MCDI_FLAG(REM_HOST, SRC_IP, encap_type); 285 MAP_FILTER_TO_MCDI_FLAG(LOC_HOST, DST_IP, encap_type); 286 MAP_FILTER_TO_MCDI_FLAG(REM_MAC, SRC_MAC, encap_type); 287 MAP_FILTER_TO_MCDI_FLAG(REM_PORT, SRC_PORT, encap_type); 288 MAP_FILTER_TO_MCDI_FLAG(LOC_MAC, DST_MAC, encap_type); 289 MAP_FILTER_TO_MCDI_FLAG(LOC_PORT, DST_PORT, encap_type); 290 MAP_FILTER_TO_MCDI_FLAG(ETHER_TYPE, ETHER_TYPE, encap_type); 291 MAP_FILTER_TO_MCDI_FLAG(IP_PROTO, IP_PROTO, encap_type); 292 /* always outer */ 293 MAP_FILTER_TO_MCDI_FLAG(INNER_VID, INNER_VLAN, false); 294 MAP_FILTER_TO_MCDI_FLAG(OUTER_VID, OUTER_VLAN, false); 295 #undef MAP_FILTER_TO_MCDI_FLAG 296 297 /* special handling for encap type, and mismatch */ 298 if (encap_type) { 299 match_flags &= ~EFX_FILTER_MATCH_ENCAP_TYPE; 300 mcdi_flags |= 301 (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 302 mcdi_flags |= (1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 303 304 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_UCAST_DST_LBN; 305 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_IFRM_UNKNOWN_MCAST_DST_LBN; 306 } else { 307 uc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_UCAST_DST_LBN; 308 mc_match = MC_CMD_FILTER_OP_EXT_IN_MATCH_UNKNOWN_MCAST_DST_LBN; 309 } 310 311 if (match_flags & EFX_FILTER_MATCH_LOC_MAC_IG) { 312 match_flags &= ~EFX_FILTER_MATCH_LOC_MAC_IG; 313 mcdi_flags |= 314 is_multicast_ether_addr(spec->loc_mac) ? 315 1 << mc_match : 316 1 << uc_match; 317 } 318 319 /* Did we map them all? */ 320 WARN_ON_ONCE(match_flags); 321 322 return mcdi_flags; 323 } 324 325 static int efx_mcdi_filter_pri(struct efx_mcdi_filter_table *table, 326 const struct efx_filter_spec *spec) 327 { 328 u32 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); 329 unsigned int match_pri; 330 331 for (match_pri = 0; 332 match_pri < table->rx_match_count; 333 match_pri++) 334 if (table->rx_match_mcdi_flags[match_pri] == mcdi_flags) 335 return match_pri; 336 337 return -EPROTONOSUPPORT; 338 } 339 340 static s32 efx_mcdi_filter_insert_locked(struct efx_nic *efx, 341 struct efx_filter_spec *spec, 342 bool replace_equal) 343 { 344 DECLARE_BITMAP(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 345 struct efx_mcdi_filter_table *table; 346 struct efx_filter_spec *saved_spec; 347 struct efx_rss_context *ctx = NULL; 348 unsigned int match_pri, hash; 349 unsigned int priv_flags; 350 bool rss_locked = false; 351 bool replacing = false; 352 unsigned int depth, i; 353 int ins_index = -1; 354 DEFINE_WAIT(wait); 355 bool is_mc_recip; 356 s32 rc; 357 358 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 359 table = efx->filter_state; 360 down_write(&table->lock); 361 362 /* For now, only support RX filters */ 363 if ((spec->flags & (EFX_FILTER_FLAG_RX | EFX_FILTER_FLAG_TX)) != 364 EFX_FILTER_FLAG_RX) { 365 rc = -EINVAL; 366 goto out_unlock; 367 } 368 369 rc = efx_mcdi_filter_pri(table, spec); 370 if (rc < 0) 371 goto out_unlock; 372 match_pri = rc; 373 374 hash = efx_filter_spec_hash(spec); 375 is_mc_recip = efx_filter_is_mc_recipient(spec); 376 if (is_mc_recip) 377 bitmap_zero(mc_rem_map, EFX_EF10_FILTER_SEARCH_LIMIT); 378 379 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 380 mutex_lock(&efx->rss_lock); 381 rss_locked = true; 382 if (spec->rss_context) 383 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 384 else 385 ctx = &efx->rss_context; 386 if (!ctx) { 387 rc = -ENOENT; 388 goto out_unlock; 389 } 390 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 391 rc = -EOPNOTSUPP; 392 goto out_unlock; 393 } 394 } 395 396 /* Find any existing filters with the same match tuple or 397 * else a free slot to insert at. 398 */ 399 for (depth = 1; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 400 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); 401 saved_spec = efx_mcdi_filter_entry_spec(table, i); 402 403 if (!saved_spec) { 404 if (ins_index < 0) 405 ins_index = i; 406 } else if (efx_filter_spec_equal(spec, saved_spec)) { 407 if (spec->priority < saved_spec->priority && 408 spec->priority != EFX_FILTER_PRI_AUTO) { 409 rc = -EPERM; 410 goto out_unlock; 411 } 412 if (!is_mc_recip) { 413 /* This is the only one */ 414 if (spec->priority == 415 saved_spec->priority && 416 !replace_equal) { 417 rc = -EEXIST; 418 goto out_unlock; 419 } 420 ins_index = i; 421 break; 422 } else if (spec->priority > 423 saved_spec->priority || 424 (spec->priority == 425 saved_spec->priority && 426 replace_equal)) { 427 if (ins_index < 0) 428 ins_index = i; 429 else 430 __set_bit(depth, mc_rem_map); 431 } 432 } 433 } 434 435 /* Once we reach the maximum search depth, use the first suitable 436 * slot, or return -EBUSY if there was none 437 */ 438 if (ins_index < 0) { 439 rc = -EBUSY; 440 goto out_unlock; 441 } 442 443 /* Create a software table entry if necessary. */ 444 saved_spec = efx_mcdi_filter_entry_spec(table, ins_index); 445 if (saved_spec) { 446 if (spec->priority == EFX_FILTER_PRI_AUTO && 447 saved_spec->priority >= EFX_FILTER_PRI_AUTO) { 448 /* Just make sure it won't be removed */ 449 if (saved_spec->priority > EFX_FILTER_PRI_AUTO) 450 saved_spec->flags |= EFX_FILTER_FLAG_RX_OVER_AUTO; 451 table->entry[ins_index].spec &= 452 ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 453 rc = ins_index; 454 goto out_unlock; 455 } 456 replacing = true; 457 priv_flags = efx_mcdi_filter_entry_flags(table, ins_index); 458 } else { 459 saved_spec = kmalloc(sizeof(*spec), GFP_ATOMIC); 460 if (!saved_spec) { 461 rc = -ENOMEM; 462 goto out_unlock; 463 } 464 *saved_spec = *spec; 465 priv_flags = 0; 466 } 467 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags); 468 469 /* Actually insert the filter on the HW */ 470 rc = efx_mcdi_filter_push(efx, spec, &table->entry[ins_index].handle, 471 ctx, replacing); 472 473 if (rc == -EINVAL && efx->must_realloc_vis) 474 /* The MC rebooted under us, causing it to reject our filter 475 * insertion as pointing to an invalid VI (spec->dmaq_id). 476 */ 477 rc = -EAGAIN; 478 479 /* Finalise the software table entry */ 480 if (rc == 0) { 481 if (replacing) { 482 /* Update the fields that may differ */ 483 if (saved_spec->priority == EFX_FILTER_PRI_AUTO) 484 saved_spec->flags |= 485 EFX_FILTER_FLAG_RX_OVER_AUTO; 486 saved_spec->priority = spec->priority; 487 saved_spec->flags &= EFX_FILTER_FLAG_RX_OVER_AUTO; 488 saved_spec->flags |= spec->flags; 489 saved_spec->rss_context = spec->rss_context; 490 saved_spec->dmaq_id = spec->dmaq_id; 491 } 492 } else if (!replacing) { 493 kfree(saved_spec); 494 saved_spec = NULL; 495 } else { 496 /* We failed to replace, so the old filter is still present. 497 * Roll back the software table to reflect this. In fact the 498 * efx_mcdi_filter_set_entry() call below will do the right 499 * thing, so nothing extra is needed here. 500 */ 501 } 502 efx_mcdi_filter_set_entry(table, ins_index, saved_spec, priv_flags); 503 504 /* Remove and finalise entries for lower-priority multicast 505 * recipients 506 */ 507 if (is_mc_recip) { 508 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 509 unsigned int depth, i; 510 511 memset(inbuf, 0, sizeof(inbuf)); 512 513 for (depth = 0; depth < EFX_EF10_FILTER_SEARCH_LIMIT; depth++) { 514 if (!test_bit(depth, mc_rem_map)) 515 continue; 516 517 i = (hash + depth) & (EFX_MCDI_FILTER_TBL_ROWS - 1); 518 saved_spec = efx_mcdi_filter_entry_spec(table, i); 519 priv_flags = efx_mcdi_filter_entry_flags(table, i); 520 521 if (rc == 0) { 522 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 523 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 524 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 525 table->entry[i].handle); 526 rc = efx_mcdi_rpc(efx, MC_CMD_FILTER_OP, 527 inbuf, sizeof(inbuf), 528 NULL, 0, NULL); 529 } 530 531 if (rc == 0) { 532 kfree(saved_spec); 533 saved_spec = NULL; 534 priv_flags = 0; 535 } 536 efx_mcdi_filter_set_entry(table, i, saved_spec, 537 priv_flags); 538 } 539 } 540 541 /* If successful, return the inserted filter ID */ 542 if (rc == 0) 543 rc = efx_mcdi_filter_make_filter_id(match_pri, ins_index); 544 545 out_unlock: 546 if (rss_locked) 547 mutex_unlock(&efx->rss_lock); 548 up_write(&table->lock); 549 return rc; 550 } 551 552 s32 efx_mcdi_filter_insert(struct efx_nic *efx, struct efx_filter_spec *spec, 553 bool replace_equal) 554 { 555 s32 ret; 556 557 down_read(&efx->filter_sem); 558 ret = efx_mcdi_filter_insert_locked(efx, spec, replace_equal); 559 up_read(&efx->filter_sem); 560 561 return ret; 562 } 563 564 /* 565 * Remove a filter. 566 * If !by_index, remove by ID 567 * If by_index, remove by index 568 * Filter ID may come from userland and must be range-checked. 569 * Caller must hold efx->filter_sem for read, and efx->filter_state->lock 570 * for write. 571 */ 572 static int efx_mcdi_filter_remove_internal(struct efx_nic *efx, 573 unsigned int priority_mask, 574 u32 filter_id, bool by_index) 575 { 576 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); 577 struct efx_mcdi_filter_table *table = efx->filter_state; 578 MCDI_DECLARE_BUF(inbuf, 579 MC_CMD_FILTER_OP_IN_HANDLE_OFST + 580 MC_CMD_FILTER_OP_IN_HANDLE_LEN); 581 struct efx_filter_spec *spec; 582 DEFINE_WAIT(wait); 583 int rc; 584 585 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 586 if (!spec || 587 (!by_index && 588 efx_mcdi_filter_pri(table, spec) != 589 efx_mcdi_filter_get_unsafe_pri(filter_id))) 590 return -ENOENT; 591 592 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO && 593 priority_mask == (1U << EFX_FILTER_PRI_AUTO)) { 594 /* Just remove flags */ 595 spec->flags &= ~EFX_FILTER_FLAG_RX_OVER_AUTO; 596 table->entry[filter_idx].spec &= ~EFX_EF10_FILTER_FLAG_AUTO_OLD; 597 return 0; 598 } 599 600 if (!(priority_mask & (1U << spec->priority))) 601 return -ENOENT; 602 603 if (spec->flags & EFX_FILTER_FLAG_RX_OVER_AUTO) { 604 /* Reset to an automatic filter */ 605 606 struct efx_filter_spec new_spec = *spec; 607 608 new_spec.priority = EFX_FILTER_PRI_AUTO; 609 new_spec.flags = (EFX_FILTER_FLAG_RX | 610 (efx_rss_active(&efx->rss_context) ? 611 EFX_FILTER_FLAG_RX_RSS : 0)); 612 new_spec.dmaq_id = 0; 613 new_spec.rss_context = 0; 614 rc = efx_mcdi_filter_push(efx, &new_spec, 615 &table->entry[filter_idx].handle, 616 &efx->rss_context, 617 true); 618 619 if (rc == 0) 620 *spec = new_spec; 621 } else { 622 /* Really remove the filter */ 623 624 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 625 efx_mcdi_filter_is_exclusive(spec) ? 626 MC_CMD_FILTER_OP_IN_OP_REMOVE : 627 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 628 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 629 table->entry[filter_idx].handle); 630 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, 631 inbuf, sizeof(inbuf), NULL, 0, NULL); 632 633 if ((rc == 0) || (rc == -ENOENT)) { 634 /* Filter removed OK or didn't actually exist */ 635 kfree(spec); 636 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0); 637 } else { 638 efx_mcdi_display_error(efx, MC_CMD_FILTER_OP, 639 MC_CMD_FILTER_OP_EXT_IN_LEN, 640 NULL, 0, rc); 641 } 642 } 643 644 return rc; 645 } 646 647 /* Remove filters that weren't renewed. */ 648 static void efx_mcdi_filter_remove_old(struct efx_nic *efx) 649 { 650 struct efx_mcdi_filter_table *table = efx->filter_state; 651 int remove_failed = 0; 652 int remove_noent = 0; 653 int rc; 654 int i; 655 656 down_write(&table->lock); 657 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { 658 if (READ_ONCE(table->entry[i].spec) & 659 EFX_EF10_FILTER_FLAG_AUTO_OLD) { 660 rc = efx_mcdi_filter_remove_internal(efx, 661 1U << EFX_FILTER_PRI_AUTO, i, true); 662 if (rc == -ENOENT) 663 remove_noent++; 664 else if (rc) 665 remove_failed++; 666 } 667 } 668 up_write(&table->lock); 669 670 if (remove_failed) 671 netif_info(efx, drv, efx->net_dev, 672 "%s: failed to remove %d filters\n", 673 __func__, remove_failed); 674 if (remove_noent) 675 netif_info(efx, drv, efx->net_dev, 676 "%s: failed to remove %d non-existent filters\n", 677 __func__, remove_noent); 678 } 679 680 int efx_mcdi_filter_remove_safe(struct efx_nic *efx, 681 enum efx_filter_priority priority, 682 u32 filter_id) 683 { 684 struct efx_mcdi_filter_table *table; 685 int rc; 686 687 down_read(&efx->filter_sem); 688 table = efx->filter_state; 689 down_write(&table->lock); 690 rc = efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id, 691 false); 692 up_write(&table->lock); 693 up_read(&efx->filter_sem); 694 return rc; 695 } 696 697 /* Caller must hold efx->filter_sem for read */ 698 static void efx_mcdi_filter_remove_unsafe(struct efx_nic *efx, 699 enum efx_filter_priority priority, 700 u32 filter_id) 701 { 702 struct efx_mcdi_filter_table *table = efx->filter_state; 703 704 if (filter_id == EFX_EF10_FILTER_ID_INVALID) 705 return; 706 707 down_write(&table->lock); 708 efx_mcdi_filter_remove_internal(efx, 1U << priority, filter_id, 709 true); 710 up_write(&table->lock); 711 } 712 713 int efx_mcdi_filter_get_safe(struct efx_nic *efx, 714 enum efx_filter_priority priority, 715 u32 filter_id, struct efx_filter_spec *spec) 716 { 717 unsigned int filter_idx = efx_mcdi_filter_get_unsafe_id(filter_id); 718 const struct efx_filter_spec *saved_spec; 719 struct efx_mcdi_filter_table *table; 720 int rc; 721 722 down_read(&efx->filter_sem); 723 table = efx->filter_state; 724 down_read(&table->lock); 725 saved_spec = efx_mcdi_filter_entry_spec(table, filter_idx); 726 if (saved_spec && saved_spec->priority == priority && 727 efx_mcdi_filter_pri(table, saved_spec) == 728 efx_mcdi_filter_get_unsafe_pri(filter_id)) { 729 *spec = *saved_spec; 730 rc = 0; 731 } else { 732 rc = -ENOENT; 733 } 734 up_read(&table->lock); 735 up_read(&efx->filter_sem); 736 return rc; 737 } 738 739 static int efx_mcdi_filter_insert_addr_list(struct efx_nic *efx, 740 struct efx_mcdi_filter_vlan *vlan, 741 bool multicast, bool rollback) 742 { 743 struct efx_mcdi_filter_table *table = efx->filter_state; 744 struct efx_mcdi_dev_addr *addr_list; 745 enum efx_filter_flags filter_flags; 746 struct efx_filter_spec spec; 747 u8 baddr[ETH_ALEN]; 748 unsigned int i, j; 749 int addr_count; 750 u16 *ids; 751 int rc; 752 753 if (multicast) { 754 addr_list = table->dev_mc_list; 755 addr_count = table->dev_mc_count; 756 ids = vlan->mc; 757 } else { 758 addr_list = table->dev_uc_list; 759 addr_count = table->dev_uc_count; 760 ids = vlan->uc; 761 } 762 763 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; 764 765 /* Insert/renew filters */ 766 for (i = 0; i < addr_count; i++) { 767 EFX_WARN_ON_PARANOID(ids[i] != EFX_EF10_FILTER_ID_INVALID); 768 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 769 efx_filter_set_eth_local(&spec, vlan->vid, addr_list[i].addr); 770 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 771 if (rc < 0) { 772 if (rollback) { 773 netif_info(efx, drv, efx->net_dev, 774 "efx_mcdi_filter_insert failed rc=%d\n", 775 rc); 776 /* Fall back to promiscuous */ 777 for (j = 0; j < i; j++) { 778 efx_mcdi_filter_remove_unsafe( 779 efx, EFX_FILTER_PRI_AUTO, 780 ids[j]); 781 ids[j] = EFX_EF10_FILTER_ID_INVALID; 782 } 783 return rc; 784 } else { 785 /* keep invalid ID, and carry on */ 786 } 787 } else { 788 ids[i] = efx_mcdi_filter_get_unsafe_id(rc); 789 } 790 } 791 792 if (multicast && rollback) { 793 /* Also need an Ethernet broadcast filter */ 794 EFX_WARN_ON_PARANOID(vlan->default_filters[EFX_EF10_BCAST] != 795 EFX_EF10_FILTER_ID_INVALID); 796 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 797 eth_broadcast_addr(baddr); 798 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 799 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 800 if (rc < 0) { 801 netif_warn(efx, drv, efx->net_dev, 802 "Broadcast filter insert failed rc=%d\n", rc); 803 /* Fall back to promiscuous */ 804 for (j = 0; j < i; j++) { 805 efx_mcdi_filter_remove_unsafe( 806 efx, EFX_FILTER_PRI_AUTO, 807 ids[j]); 808 ids[j] = EFX_EF10_FILTER_ID_INVALID; 809 } 810 return rc; 811 } else { 812 vlan->default_filters[EFX_EF10_BCAST] = 813 efx_mcdi_filter_get_unsafe_id(rc); 814 } 815 } 816 817 return 0; 818 } 819 820 static int efx_mcdi_filter_insert_def(struct efx_nic *efx, 821 struct efx_mcdi_filter_vlan *vlan, 822 enum efx_encap_type encap_type, 823 bool multicast, bool rollback) 824 { 825 struct efx_mcdi_filter_table *table = efx->filter_state; 826 enum efx_filter_flags filter_flags; 827 struct efx_filter_spec spec; 828 u8 baddr[ETH_ALEN]; 829 int rc; 830 u16 *id; 831 832 filter_flags = efx_rss_active(&efx->rss_context) ? EFX_FILTER_FLAG_RX_RSS : 0; 833 834 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, filter_flags, 0); 835 836 if (multicast) 837 efx_filter_set_mc_def(&spec); 838 else 839 efx_filter_set_uc_def(&spec); 840 841 if (encap_type) { 842 if (efx_has_cap(efx, VXLAN_NVGRE)) 843 efx_filter_set_encap_type(&spec, encap_type); 844 else 845 /* 846 * don't insert encap filters on non-supporting 847 * platforms. ID will be left as INVALID. 848 */ 849 return 0; 850 } 851 852 if (vlan->vid != EFX_FILTER_VID_UNSPEC) 853 efx_filter_set_eth_local(&spec, vlan->vid, NULL); 854 855 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 856 if (rc < 0) { 857 const char *um = multicast ? "Multicast" : "Unicast"; 858 const char *encap_name = ""; 859 const char *encap_ipv = ""; 860 861 if ((encap_type & EFX_ENCAP_TYPES_MASK) == 862 EFX_ENCAP_TYPE_VXLAN) 863 encap_name = "VXLAN "; 864 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 865 EFX_ENCAP_TYPE_NVGRE) 866 encap_name = "NVGRE "; 867 else if ((encap_type & EFX_ENCAP_TYPES_MASK) == 868 EFX_ENCAP_TYPE_GENEVE) 869 encap_name = "GENEVE "; 870 if (encap_type & EFX_ENCAP_FLAG_IPV6) 871 encap_ipv = "IPv6 "; 872 else if (encap_type) 873 encap_ipv = "IPv4 "; 874 875 /* 876 * unprivileged functions can't insert mismatch filters 877 * for encapsulated or unicast traffic, so downgrade 878 * those warnings to debug. 879 */ 880 netif_cond_dbg(efx, drv, efx->net_dev, 881 rc == -EPERM && (encap_type || !multicast), warn, 882 "%s%s%s mismatch filter insert failed rc=%d\n", 883 encap_name, encap_ipv, um, rc); 884 } else if (multicast) { 885 /* mapping from encap types to default filter IDs (multicast) */ 886 static enum efx_mcdi_filter_default_filters map[] = { 887 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_MCDEF, 888 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_MCDEF, 889 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_MCDEF, 890 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_MCDEF, 891 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 892 EFX_EF10_VXLAN6_MCDEF, 893 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 894 EFX_EF10_NVGRE6_MCDEF, 895 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 896 EFX_EF10_GENEVE6_MCDEF, 897 }; 898 899 /* quick bounds check (BCAST result impossible) */ 900 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 901 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 902 WARN_ON(1); 903 return -EINVAL; 904 } 905 /* then follow map */ 906 id = &vlan->default_filters[map[encap_type]]; 907 908 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 909 *id = efx_mcdi_filter_get_unsafe_id(rc); 910 if (!table->mc_chaining && !encap_type) { 911 /* Also need an Ethernet broadcast filter */ 912 efx_filter_init_rx(&spec, EFX_FILTER_PRI_AUTO, 913 filter_flags, 0); 914 eth_broadcast_addr(baddr); 915 efx_filter_set_eth_local(&spec, vlan->vid, baddr); 916 rc = efx_mcdi_filter_insert_locked(efx, &spec, true); 917 if (rc < 0) { 918 netif_warn(efx, drv, efx->net_dev, 919 "Broadcast filter insert failed rc=%d\n", 920 rc); 921 if (rollback) { 922 /* Roll back the mc_def filter */ 923 efx_mcdi_filter_remove_unsafe( 924 efx, EFX_FILTER_PRI_AUTO, 925 *id); 926 *id = EFX_EF10_FILTER_ID_INVALID; 927 return rc; 928 } 929 } else { 930 EFX_WARN_ON_PARANOID( 931 vlan->default_filters[EFX_EF10_BCAST] != 932 EFX_EF10_FILTER_ID_INVALID); 933 vlan->default_filters[EFX_EF10_BCAST] = 934 efx_mcdi_filter_get_unsafe_id(rc); 935 } 936 } 937 rc = 0; 938 } else { 939 /* mapping from encap types to default filter IDs (unicast) */ 940 static enum efx_mcdi_filter_default_filters map[] = { 941 [EFX_ENCAP_TYPE_NONE] = EFX_EF10_UCDEF, 942 [EFX_ENCAP_TYPE_VXLAN] = EFX_EF10_VXLAN4_UCDEF, 943 [EFX_ENCAP_TYPE_NVGRE] = EFX_EF10_NVGRE4_UCDEF, 944 [EFX_ENCAP_TYPE_GENEVE] = EFX_EF10_GENEVE4_UCDEF, 945 [EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6] = 946 EFX_EF10_VXLAN6_UCDEF, 947 [EFX_ENCAP_TYPE_NVGRE | EFX_ENCAP_FLAG_IPV6] = 948 EFX_EF10_NVGRE6_UCDEF, 949 [EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6] = 950 EFX_EF10_GENEVE6_UCDEF, 951 }; 952 953 /* quick bounds check (BCAST result impossible) */ 954 BUILD_BUG_ON(EFX_EF10_BCAST != 0); 955 if (encap_type >= ARRAY_SIZE(map) || map[encap_type] == 0) { 956 WARN_ON(1); 957 return -EINVAL; 958 } 959 /* then follow map */ 960 id = &vlan->default_filters[map[encap_type]]; 961 EFX_WARN_ON_PARANOID(*id != EFX_EF10_FILTER_ID_INVALID); 962 *id = rc; 963 rc = 0; 964 } 965 return rc; 966 } 967 968 /* 969 * Caller must hold efx->filter_sem for read if race against 970 * efx_mcdi_filter_table_remove() is possible 971 */ 972 static void efx_mcdi_filter_vlan_sync_rx_mode(struct efx_nic *efx, 973 struct efx_mcdi_filter_vlan *vlan) 974 { 975 struct efx_mcdi_filter_table *table = efx->filter_state; 976 977 /* 978 * Do not install unspecified VID if VLAN filtering is enabled. 979 * Do not install all specified VIDs if VLAN filtering is disabled. 980 */ 981 if ((vlan->vid == EFX_FILTER_VID_UNSPEC) == table->vlan_filter) 982 return; 983 984 /* Insert/renew unicast filters */ 985 if (table->uc_promisc) { 986 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NONE, 987 false, false); 988 efx_mcdi_filter_insert_addr_list(efx, vlan, false, false); 989 } else { 990 /* 991 * If any of the filters failed to insert, fall back to 992 * promiscuous mode - add in the uc_def filter. But keep 993 * our individual unicast filters. 994 */ 995 if (efx_mcdi_filter_insert_addr_list(efx, vlan, false, false)) 996 efx_mcdi_filter_insert_def(efx, vlan, 997 EFX_ENCAP_TYPE_NONE, 998 false, false); 999 } 1000 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 1001 false, false); 1002 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 1003 EFX_ENCAP_FLAG_IPV6, 1004 false, false); 1005 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 1006 false, false); 1007 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 1008 EFX_ENCAP_FLAG_IPV6, 1009 false, false); 1010 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 1011 false, false); 1012 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 1013 EFX_ENCAP_FLAG_IPV6, 1014 false, false); 1015 1016 /* 1017 * Insert/renew multicast filters 1018 * 1019 * If changing promiscuous state with cascaded multicast filters, remove 1020 * old filters first, so that packets are dropped rather than duplicated 1021 */ 1022 if (table->mc_chaining && table->mc_promisc_last != table->mc_promisc) 1023 efx_mcdi_filter_remove_old(efx); 1024 if (table->mc_promisc) { 1025 if (table->mc_chaining) { 1026 /* 1027 * If we failed to insert promiscuous filters, rollback 1028 * and fall back to individual multicast filters 1029 */ 1030 if (efx_mcdi_filter_insert_def(efx, vlan, 1031 EFX_ENCAP_TYPE_NONE, 1032 true, true)) { 1033 /* Changing promisc state, so remove old filters */ 1034 efx_mcdi_filter_remove_old(efx); 1035 efx_mcdi_filter_insert_addr_list(efx, vlan, 1036 true, false); 1037 } 1038 } else { 1039 /* 1040 * If we failed to insert promiscuous filters, don't 1041 * rollback. Regardless, also insert the mc_list, 1042 * unless it's incomplete due to overflow 1043 */ 1044 efx_mcdi_filter_insert_def(efx, vlan, 1045 EFX_ENCAP_TYPE_NONE, 1046 true, false); 1047 if (!table->mc_overflow) 1048 efx_mcdi_filter_insert_addr_list(efx, vlan, 1049 true, false); 1050 } 1051 } else { 1052 /* 1053 * If any filters failed to insert, rollback and fall back to 1054 * promiscuous mode - mc_def filter and maybe broadcast. If 1055 * that fails, roll back again and insert as many of our 1056 * individual multicast filters as we can. 1057 */ 1058 if (efx_mcdi_filter_insert_addr_list(efx, vlan, true, true)) { 1059 /* Changing promisc state, so remove old filters */ 1060 if (table->mc_chaining) 1061 efx_mcdi_filter_remove_old(efx); 1062 if (efx_mcdi_filter_insert_def(efx, vlan, 1063 EFX_ENCAP_TYPE_NONE, 1064 true, true)) 1065 efx_mcdi_filter_insert_addr_list(efx, vlan, 1066 true, false); 1067 } 1068 } 1069 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN, 1070 true, false); 1071 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_VXLAN | 1072 EFX_ENCAP_FLAG_IPV6, 1073 true, false); 1074 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE, 1075 true, false); 1076 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_NVGRE | 1077 EFX_ENCAP_FLAG_IPV6, 1078 true, false); 1079 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE, 1080 true, false); 1081 efx_mcdi_filter_insert_def(efx, vlan, EFX_ENCAP_TYPE_GENEVE | 1082 EFX_ENCAP_FLAG_IPV6, 1083 true, false); 1084 } 1085 1086 int efx_mcdi_filter_clear_rx(struct efx_nic *efx, 1087 enum efx_filter_priority priority) 1088 { 1089 struct efx_mcdi_filter_table *table; 1090 unsigned int priority_mask; 1091 unsigned int i; 1092 int rc; 1093 1094 priority_mask = (((1U << (priority + 1)) - 1) & 1095 ~(1U << EFX_FILTER_PRI_AUTO)); 1096 1097 down_read(&efx->filter_sem); 1098 table = efx->filter_state; 1099 down_write(&table->lock); 1100 for (i = 0; i < EFX_MCDI_FILTER_TBL_ROWS; i++) { 1101 rc = efx_mcdi_filter_remove_internal(efx, priority_mask, 1102 i, true); 1103 if (rc && rc != -ENOENT) 1104 break; 1105 rc = 0; 1106 } 1107 1108 up_write(&table->lock); 1109 up_read(&efx->filter_sem); 1110 return rc; 1111 } 1112 1113 u32 efx_mcdi_filter_count_rx_used(struct efx_nic *efx, 1114 enum efx_filter_priority priority) 1115 { 1116 struct efx_mcdi_filter_table *table; 1117 unsigned int filter_idx; 1118 s32 count = 0; 1119 1120 down_read(&efx->filter_sem); 1121 table = efx->filter_state; 1122 down_read(&table->lock); 1123 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1124 if (table->entry[filter_idx].spec && 1125 efx_mcdi_filter_entry_spec(table, filter_idx)->priority == 1126 priority) 1127 ++count; 1128 } 1129 up_read(&table->lock); 1130 up_read(&efx->filter_sem); 1131 return count; 1132 } 1133 1134 u32 efx_mcdi_filter_get_rx_id_limit(struct efx_nic *efx) 1135 { 1136 struct efx_mcdi_filter_table *table = efx->filter_state; 1137 1138 return table->rx_match_count * EFX_MCDI_FILTER_TBL_ROWS * 2; 1139 } 1140 1141 s32 efx_mcdi_filter_get_rx_ids(struct efx_nic *efx, 1142 enum efx_filter_priority priority, 1143 u32 *buf, u32 size) 1144 { 1145 struct efx_mcdi_filter_table *table; 1146 struct efx_filter_spec *spec; 1147 unsigned int filter_idx; 1148 s32 count = 0; 1149 1150 down_read(&efx->filter_sem); 1151 table = efx->filter_state; 1152 down_read(&table->lock); 1153 1154 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1155 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1156 if (spec && spec->priority == priority) { 1157 if (count == size) { 1158 count = -EMSGSIZE; 1159 break; 1160 } 1161 buf[count++] = 1162 efx_mcdi_filter_make_filter_id( 1163 efx_mcdi_filter_pri(table, spec), 1164 filter_idx); 1165 } 1166 } 1167 up_read(&table->lock); 1168 up_read(&efx->filter_sem); 1169 return count; 1170 } 1171 1172 static int efx_mcdi_filter_match_flags_from_mcdi(bool encap, u32 mcdi_flags) 1173 { 1174 int match_flags = 0; 1175 1176 #define MAP_FLAG(gen_flag, mcdi_field) do { \ 1177 u32 old_mcdi_flags = mcdi_flags; \ 1178 mcdi_flags &= ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ ## \ 1179 mcdi_field ## _LBN); \ 1180 if (mcdi_flags != old_mcdi_flags) \ 1181 match_flags |= EFX_FILTER_MATCH_ ## gen_flag; \ 1182 } while (0) 1183 1184 if (encap) { 1185 /* encap filters must specify encap type */ 1186 match_flags |= EFX_FILTER_MATCH_ENCAP_TYPE; 1187 /* and imply ethertype and ip proto */ 1188 mcdi_flags &= 1189 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_IP_PROTO_LBN); 1190 mcdi_flags &= 1191 ~(1 << MC_CMD_FILTER_OP_EXT_IN_MATCH_ETHER_TYPE_LBN); 1192 /* VLAN tags refer to the outer packet */ 1193 MAP_FLAG(INNER_VID, INNER_VLAN); 1194 MAP_FLAG(OUTER_VID, OUTER_VLAN); 1195 /* everything else refers to the inner packet */ 1196 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_UCAST_DST); 1197 MAP_FLAG(LOC_MAC_IG, IFRM_UNKNOWN_MCAST_DST); 1198 MAP_FLAG(REM_HOST, IFRM_SRC_IP); 1199 MAP_FLAG(LOC_HOST, IFRM_DST_IP); 1200 MAP_FLAG(REM_MAC, IFRM_SRC_MAC); 1201 MAP_FLAG(REM_PORT, IFRM_SRC_PORT); 1202 MAP_FLAG(LOC_MAC, IFRM_DST_MAC); 1203 MAP_FLAG(LOC_PORT, IFRM_DST_PORT); 1204 MAP_FLAG(ETHER_TYPE, IFRM_ETHER_TYPE); 1205 MAP_FLAG(IP_PROTO, IFRM_IP_PROTO); 1206 } else { 1207 MAP_FLAG(LOC_MAC_IG, UNKNOWN_UCAST_DST); 1208 MAP_FLAG(LOC_MAC_IG, UNKNOWN_MCAST_DST); 1209 MAP_FLAG(REM_HOST, SRC_IP); 1210 MAP_FLAG(LOC_HOST, DST_IP); 1211 MAP_FLAG(REM_MAC, SRC_MAC); 1212 MAP_FLAG(REM_PORT, SRC_PORT); 1213 MAP_FLAG(LOC_MAC, DST_MAC); 1214 MAP_FLAG(LOC_PORT, DST_PORT); 1215 MAP_FLAG(ETHER_TYPE, ETHER_TYPE); 1216 MAP_FLAG(INNER_VID, INNER_VLAN); 1217 MAP_FLAG(OUTER_VID, OUTER_VLAN); 1218 MAP_FLAG(IP_PROTO, IP_PROTO); 1219 } 1220 #undef MAP_FLAG 1221 1222 /* Did we map them all? */ 1223 if (mcdi_flags) 1224 return -EINVAL; 1225 1226 return match_flags; 1227 } 1228 1229 bool efx_mcdi_filter_match_supported(struct efx_mcdi_filter_table *table, 1230 bool encap, 1231 enum efx_filter_match_flags match_flags) 1232 { 1233 unsigned int match_pri; 1234 int mf; 1235 1236 for (match_pri = 0; 1237 match_pri < table->rx_match_count; 1238 match_pri++) { 1239 mf = efx_mcdi_filter_match_flags_from_mcdi(encap, 1240 table->rx_match_mcdi_flags[match_pri]); 1241 if (mf == match_flags) 1242 return true; 1243 } 1244 1245 return false; 1246 } 1247 1248 static int 1249 efx_mcdi_filter_table_probe_matches(struct efx_nic *efx, 1250 struct efx_mcdi_filter_table *table, 1251 bool encap) 1252 { 1253 MCDI_DECLARE_BUF(inbuf, MC_CMD_GET_PARSER_DISP_INFO_IN_LEN); 1254 MCDI_DECLARE_BUF(outbuf, MC_CMD_GET_PARSER_DISP_INFO_OUT_LENMAX); 1255 unsigned int pd_match_pri, pd_match_count; 1256 size_t outlen; 1257 int rc; 1258 1259 /* Find out which RX filter types are supported, and their priorities */ 1260 MCDI_SET_DWORD(inbuf, GET_PARSER_DISP_INFO_IN_OP, 1261 encap ? 1262 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_ENCAP_RX_MATCHES : 1263 MC_CMD_GET_PARSER_DISP_INFO_IN_OP_GET_SUPPORTED_RX_MATCHES); 1264 rc = efx_mcdi_rpc(efx, MC_CMD_GET_PARSER_DISP_INFO, 1265 inbuf, sizeof(inbuf), outbuf, sizeof(outbuf), 1266 &outlen); 1267 if (rc) 1268 return rc; 1269 1270 pd_match_count = MCDI_VAR_ARRAY_LEN( 1271 outlen, GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES); 1272 1273 for (pd_match_pri = 0; pd_match_pri < pd_match_count; pd_match_pri++) { 1274 u32 mcdi_flags = 1275 MCDI_ARRAY_DWORD( 1276 outbuf, 1277 GET_PARSER_DISP_INFO_OUT_SUPPORTED_MATCHES, 1278 pd_match_pri); 1279 rc = efx_mcdi_filter_match_flags_from_mcdi(encap, mcdi_flags); 1280 if (rc < 0) { 1281 netif_dbg(efx, probe, efx->net_dev, 1282 "%s: fw flags %#x pri %u not supported in driver\n", 1283 __func__, mcdi_flags, pd_match_pri); 1284 } else { 1285 netif_dbg(efx, probe, efx->net_dev, 1286 "%s: fw flags %#x pri %u supported as driver flags %#x pri %u\n", 1287 __func__, mcdi_flags, pd_match_pri, 1288 rc, table->rx_match_count); 1289 table->rx_match_mcdi_flags[table->rx_match_count] = mcdi_flags; 1290 table->rx_match_count++; 1291 } 1292 } 1293 1294 return 0; 1295 } 1296 1297 int efx_mcdi_filter_table_probe(struct efx_nic *efx, bool multicast_chaining) 1298 { 1299 struct net_device *net_dev = efx->net_dev; 1300 struct efx_mcdi_filter_table *table; 1301 int rc; 1302 1303 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1304 return -EINVAL; 1305 1306 if (efx->filter_state) /* already probed */ 1307 return 0; 1308 1309 table = kzalloc(sizeof(*table), GFP_KERNEL); 1310 if (!table) 1311 return -ENOMEM; 1312 1313 table->mc_chaining = multicast_chaining; 1314 table->rx_match_count = 0; 1315 rc = efx_mcdi_filter_table_probe_matches(efx, table, false); 1316 if (rc) 1317 goto fail; 1318 if (efx_has_cap(efx, VXLAN_NVGRE)) 1319 rc = efx_mcdi_filter_table_probe_matches(efx, table, true); 1320 if (rc) 1321 goto fail; 1322 if ((efx_supported_features(efx) & NETIF_F_HW_VLAN_CTAG_FILTER) && 1323 !(efx_mcdi_filter_match_supported(table, false, 1324 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC)) && 1325 efx_mcdi_filter_match_supported(table, false, 1326 (EFX_FILTER_MATCH_OUTER_VID | EFX_FILTER_MATCH_LOC_MAC_IG)))) { 1327 netif_info(efx, probe, net_dev, 1328 "VLAN filters are not supported in this firmware variant\n"); 1329 net_dev->features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1330 efx->fixed_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1331 net_dev->hw_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 1332 } 1333 1334 table->entry = vzalloc(array_size(EFX_MCDI_FILTER_TBL_ROWS, 1335 sizeof(*table->entry))); 1336 if (!table->entry) { 1337 rc = -ENOMEM; 1338 goto fail; 1339 } 1340 1341 table->mc_promisc_last = false; 1342 table->vlan_filter = 1343 !!(efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 1344 INIT_LIST_HEAD(&table->vlan_list); 1345 init_rwsem(&table->lock); 1346 1347 efx->filter_state = table; 1348 1349 return 0; 1350 fail: 1351 kfree(table); 1352 return rc; 1353 } 1354 1355 void efx_mcdi_filter_table_reset_mc_allocations(struct efx_nic *efx) 1356 { 1357 struct efx_mcdi_filter_table *table = efx->filter_state; 1358 1359 if (table) { 1360 table->must_restore_filters = true; 1361 table->must_restore_rss_contexts = true; 1362 } 1363 } 1364 1365 /* 1366 * Caller must hold efx->filter_sem for read if race against 1367 * efx_mcdi_filter_table_remove() is possible 1368 */ 1369 void efx_mcdi_filter_table_restore(struct efx_nic *efx) 1370 { 1371 struct efx_mcdi_filter_table *table = efx->filter_state; 1372 unsigned int invalid_filters = 0, failed = 0; 1373 struct efx_mcdi_filter_vlan *vlan; 1374 struct efx_filter_spec *spec; 1375 struct efx_rss_context *ctx; 1376 unsigned int filter_idx; 1377 u32 mcdi_flags; 1378 int match_pri; 1379 int rc, i; 1380 1381 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 1382 1383 if (!table || !table->must_restore_filters) 1384 return; 1385 1386 down_write(&table->lock); 1387 mutex_lock(&efx->rss_lock); 1388 1389 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1390 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1391 if (!spec) 1392 continue; 1393 1394 mcdi_flags = efx_mcdi_filter_mcdi_flags_from_spec(spec); 1395 match_pri = 0; 1396 while (match_pri < table->rx_match_count && 1397 table->rx_match_mcdi_flags[match_pri] != mcdi_flags) 1398 ++match_pri; 1399 if (match_pri >= table->rx_match_count) { 1400 invalid_filters++; 1401 goto not_restored; 1402 } 1403 if (spec->rss_context) 1404 ctx = efx_find_rss_context_entry(efx, spec->rss_context); 1405 else 1406 ctx = &efx->rss_context; 1407 if (spec->flags & EFX_FILTER_FLAG_RX_RSS) { 1408 if (!ctx) { 1409 netif_warn(efx, drv, efx->net_dev, 1410 "Warning: unable to restore a filter with nonexistent RSS context %u.\n", 1411 spec->rss_context); 1412 invalid_filters++; 1413 goto not_restored; 1414 } 1415 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 1416 netif_warn(efx, drv, efx->net_dev, 1417 "Warning: unable to restore a filter with RSS context %u as it was not created.\n", 1418 spec->rss_context); 1419 invalid_filters++; 1420 goto not_restored; 1421 } 1422 } 1423 1424 rc = efx_mcdi_filter_push(efx, spec, 1425 &table->entry[filter_idx].handle, 1426 ctx, false); 1427 if (rc) 1428 failed++; 1429 1430 if (rc) { 1431 not_restored: 1432 list_for_each_entry(vlan, &table->vlan_list, list) 1433 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; ++i) 1434 if (vlan->default_filters[i] == filter_idx) 1435 vlan->default_filters[i] = 1436 EFX_EF10_FILTER_ID_INVALID; 1437 1438 kfree(spec); 1439 efx_mcdi_filter_set_entry(table, filter_idx, NULL, 0); 1440 } 1441 } 1442 1443 mutex_unlock(&efx->rss_lock); 1444 up_write(&table->lock); 1445 1446 /* 1447 * This can happen validly if the MC's capabilities have changed, so 1448 * is not an error. 1449 */ 1450 if (invalid_filters) 1451 netif_dbg(efx, drv, efx->net_dev, 1452 "Did not restore %u filters that are now unsupported.\n", 1453 invalid_filters); 1454 1455 if (failed) 1456 netif_err(efx, hw, efx->net_dev, 1457 "unable to restore %u filters\n", failed); 1458 else 1459 table->must_restore_filters = false; 1460 } 1461 1462 void efx_mcdi_filter_table_down(struct efx_nic *efx) 1463 { 1464 struct efx_mcdi_filter_table *table = efx->filter_state; 1465 MCDI_DECLARE_BUF(inbuf, MC_CMD_FILTER_OP_EXT_IN_LEN); 1466 struct efx_filter_spec *spec; 1467 unsigned int filter_idx; 1468 int rc; 1469 1470 if (!table) 1471 return; 1472 1473 efx_mcdi_filter_cleanup_vlans(efx); 1474 1475 for (filter_idx = 0; filter_idx < EFX_MCDI_FILTER_TBL_ROWS; filter_idx++) { 1476 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1477 if (!spec) 1478 continue; 1479 1480 MCDI_SET_DWORD(inbuf, FILTER_OP_IN_OP, 1481 efx_mcdi_filter_is_exclusive(spec) ? 1482 MC_CMD_FILTER_OP_IN_OP_REMOVE : 1483 MC_CMD_FILTER_OP_IN_OP_UNSUBSCRIBE); 1484 MCDI_SET_QWORD(inbuf, FILTER_OP_IN_HANDLE, 1485 table->entry[filter_idx].handle); 1486 rc = efx_mcdi_rpc_quiet(efx, MC_CMD_FILTER_OP, inbuf, 1487 sizeof(inbuf), NULL, 0, NULL); 1488 if (rc) 1489 netif_info(efx, drv, efx->net_dev, 1490 "%s: filter %04x remove failed\n", 1491 __func__, filter_idx); 1492 kfree(spec); 1493 } 1494 } 1495 1496 void efx_mcdi_filter_table_remove(struct efx_nic *efx) 1497 { 1498 struct efx_mcdi_filter_table *table = efx->filter_state; 1499 1500 efx_mcdi_filter_table_down(efx); 1501 1502 efx->filter_state = NULL; 1503 /* 1504 * If we were called without locking, then it's not safe to free 1505 * the table as others might be using it. So we just WARN, leak 1506 * the memory, and potentially get an inconsistent filter table 1507 * state. 1508 * This should never actually happen. 1509 */ 1510 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1511 return; 1512 1513 if (!table) 1514 return; 1515 1516 vfree(table->entry); 1517 kfree(table); 1518 } 1519 1520 static void efx_mcdi_filter_mark_one_old(struct efx_nic *efx, uint16_t *id) 1521 { 1522 struct efx_mcdi_filter_table *table = efx->filter_state; 1523 unsigned int filter_idx; 1524 1525 efx_rwsem_assert_write_locked(&table->lock); 1526 1527 if (*id != EFX_EF10_FILTER_ID_INVALID) { 1528 filter_idx = efx_mcdi_filter_get_unsafe_id(*id); 1529 if (!table->entry[filter_idx].spec) 1530 netif_dbg(efx, drv, efx->net_dev, 1531 "marked null spec old %04x:%04x\n", *id, 1532 filter_idx); 1533 table->entry[filter_idx].spec |= EFX_EF10_FILTER_FLAG_AUTO_OLD; 1534 *id = EFX_EF10_FILTER_ID_INVALID; 1535 } 1536 } 1537 1538 /* Mark old per-VLAN filters that may need to be removed */ 1539 static void _efx_mcdi_filter_vlan_mark_old(struct efx_nic *efx, 1540 struct efx_mcdi_filter_vlan *vlan) 1541 { 1542 struct efx_mcdi_filter_table *table = efx->filter_state; 1543 unsigned int i; 1544 1545 for (i = 0; i < table->dev_uc_count; i++) 1546 efx_mcdi_filter_mark_one_old(efx, &vlan->uc[i]); 1547 for (i = 0; i < table->dev_mc_count; i++) 1548 efx_mcdi_filter_mark_one_old(efx, &vlan->mc[i]); 1549 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1550 efx_mcdi_filter_mark_one_old(efx, &vlan->default_filters[i]); 1551 } 1552 1553 /* 1554 * Mark old filters that may need to be removed. 1555 * Caller must hold efx->filter_sem for read if race against 1556 * efx_mcdi_filter_table_remove() is possible 1557 */ 1558 static void efx_mcdi_filter_mark_old(struct efx_nic *efx) 1559 { 1560 struct efx_mcdi_filter_table *table = efx->filter_state; 1561 struct efx_mcdi_filter_vlan *vlan; 1562 1563 down_write(&table->lock); 1564 list_for_each_entry(vlan, &table->vlan_list, list) 1565 _efx_mcdi_filter_vlan_mark_old(efx, vlan); 1566 up_write(&table->lock); 1567 } 1568 1569 int efx_mcdi_filter_add_vlan(struct efx_nic *efx, u16 vid) 1570 { 1571 struct efx_mcdi_filter_table *table = efx->filter_state; 1572 struct efx_mcdi_filter_vlan *vlan; 1573 unsigned int i; 1574 1575 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1576 return -EINVAL; 1577 1578 vlan = efx_mcdi_filter_find_vlan(efx, vid); 1579 if (WARN_ON(vlan)) { 1580 netif_err(efx, drv, efx->net_dev, 1581 "VLAN %u already added\n", vid); 1582 return -EALREADY; 1583 } 1584 1585 vlan = kzalloc(sizeof(*vlan), GFP_KERNEL); 1586 if (!vlan) 1587 return -ENOMEM; 1588 1589 vlan->vid = vid; 1590 1591 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 1592 vlan->uc[i] = EFX_EF10_FILTER_ID_INVALID; 1593 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 1594 vlan->mc[i] = EFX_EF10_FILTER_ID_INVALID; 1595 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1596 vlan->default_filters[i] = EFX_EF10_FILTER_ID_INVALID; 1597 1598 list_add_tail(&vlan->list, &table->vlan_list); 1599 1600 if (efx_dev_registered(efx)) 1601 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); 1602 1603 return 0; 1604 } 1605 1606 static void efx_mcdi_filter_del_vlan_internal(struct efx_nic *efx, 1607 struct efx_mcdi_filter_vlan *vlan) 1608 { 1609 unsigned int i; 1610 1611 /* See comment in efx_mcdi_filter_table_remove() */ 1612 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1613 return; 1614 1615 list_del(&vlan->list); 1616 1617 for (i = 0; i < ARRAY_SIZE(vlan->uc); i++) 1618 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1619 vlan->uc[i]); 1620 for (i = 0; i < ARRAY_SIZE(vlan->mc); i++) 1621 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1622 vlan->mc[i]); 1623 for (i = 0; i < EFX_EF10_NUM_DEFAULT_FILTERS; i++) 1624 if (vlan->default_filters[i] != EFX_EF10_FILTER_ID_INVALID) 1625 efx_mcdi_filter_remove_unsafe(efx, EFX_FILTER_PRI_AUTO, 1626 vlan->default_filters[i]); 1627 1628 kfree(vlan); 1629 } 1630 1631 void efx_mcdi_filter_del_vlan(struct efx_nic *efx, u16 vid) 1632 { 1633 struct efx_mcdi_filter_vlan *vlan; 1634 1635 /* See comment in efx_mcdi_filter_table_remove() */ 1636 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1637 return; 1638 1639 vlan = efx_mcdi_filter_find_vlan(efx, vid); 1640 if (!vlan) { 1641 netif_err(efx, drv, efx->net_dev, 1642 "VLAN %u not found in filter state\n", vid); 1643 return; 1644 } 1645 1646 efx_mcdi_filter_del_vlan_internal(efx, vlan); 1647 } 1648 1649 struct efx_mcdi_filter_vlan *efx_mcdi_filter_find_vlan(struct efx_nic *efx, 1650 u16 vid) 1651 { 1652 struct efx_mcdi_filter_table *table = efx->filter_state; 1653 struct efx_mcdi_filter_vlan *vlan; 1654 1655 WARN_ON(!rwsem_is_locked(&efx->filter_sem)); 1656 1657 list_for_each_entry(vlan, &table->vlan_list, list) { 1658 if (vlan->vid == vid) 1659 return vlan; 1660 } 1661 1662 return NULL; 1663 } 1664 1665 void efx_mcdi_filter_cleanup_vlans(struct efx_nic *efx) 1666 { 1667 struct efx_mcdi_filter_table *table = efx->filter_state; 1668 struct efx_mcdi_filter_vlan *vlan, *next_vlan; 1669 1670 /* See comment in efx_mcdi_filter_table_remove() */ 1671 if (!efx_rwsem_assert_write_locked(&efx->filter_sem)) 1672 return; 1673 1674 if (!table) 1675 return; 1676 1677 list_for_each_entry_safe(vlan, next_vlan, &table->vlan_list, list) 1678 efx_mcdi_filter_del_vlan_internal(efx, vlan); 1679 } 1680 1681 static void efx_mcdi_filter_uc_addr_list(struct efx_nic *efx) 1682 { 1683 struct efx_mcdi_filter_table *table = efx->filter_state; 1684 struct net_device *net_dev = efx->net_dev; 1685 struct netdev_hw_addr *uc; 1686 unsigned int i; 1687 1688 table->uc_promisc = !!(net_dev->flags & IFF_PROMISC); 1689 ether_addr_copy(table->dev_uc_list[0].addr, net_dev->dev_addr); 1690 i = 1; 1691 netdev_for_each_uc_addr(uc, net_dev) { 1692 if (i >= EFX_EF10_FILTER_DEV_UC_MAX) { 1693 table->uc_promisc = true; 1694 break; 1695 } 1696 ether_addr_copy(table->dev_uc_list[i].addr, uc->addr); 1697 i++; 1698 } 1699 1700 table->dev_uc_count = i; 1701 } 1702 1703 static void efx_mcdi_filter_mc_addr_list(struct efx_nic *efx) 1704 { 1705 struct efx_mcdi_filter_table *table = efx->filter_state; 1706 struct net_device *net_dev = efx->net_dev; 1707 struct netdev_hw_addr *mc; 1708 unsigned int i; 1709 1710 table->mc_overflow = false; 1711 table->mc_promisc = !!(net_dev->flags & (IFF_PROMISC | IFF_ALLMULTI)); 1712 1713 i = 0; 1714 netdev_for_each_mc_addr(mc, net_dev) { 1715 if (i >= EFX_EF10_FILTER_DEV_MC_MAX) { 1716 table->mc_promisc = true; 1717 table->mc_overflow = true; 1718 break; 1719 } 1720 ether_addr_copy(table->dev_mc_list[i].addr, mc->addr); 1721 i++; 1722 } 1723 1724 table->dev_mc_count = i; 1725 } 1726 1727 /* 1728 * Caller must hold efx->filter_sem for read if race against 1729 * efx_mcdi_filter_table_remove() is possible 1730 */ 1731 void efx_mcdi_filter_sync_rx_mode(struct efx_nic *efx) 1732 { 1733 struct efx_mcdi_filter_table *table = efx->filter_state; 1734 struct net_device *net_dev = efx->net_dev; 1735 struct efx_mcdi_filter_vlan *vlan; 1736 bool vlan_filter; 1737 1738 if (!efx_dev_registered(efx)) 1739 return; 1740 1741 if (!table) 1742 return; 1743 1744 efx_mcdi_filter_mark_old(efx); 1745 1746 /* 1747 * Copy/convert the address lists; add the primary station 1748 * address and broadcast address 1749 */ 1750 netif_addr_lock_bh(net_dev); 1751 efx_mcdi_filter_uc_addr_list(efx); 1752 efx_mcdi_filter_mc_addr_list(efx); 1753 netif_addr_unlock_bh(net_dev); 1754 1755 /* 1756 * If VLAN filtering changes, all old filters are finally removed. 1757 * Do it in advance to avoid conflicts for unicast untagged and 1758 * VLAN 0 tagged filters. 1759 */ 1760 vlan_filter = !!(net_dev->features & NETIF_F_HW_VLAN_CTAG_FILTER); 1761 if (table->vlan_filter != vlan_filter) { 1762 table->vlan_filter = vlan_filter; 1763 efx_mcdi_filter_remove_old(efx); 1764 } 1765 1766 list_for_each_entry(vlan, &table->vlan_list, list) 1767 efx_mcdi_filter_vlan_sync_rx_mode(efx, vlan); 1768 1769 efx_mcdi_filter_remove_old(efx); 1770 table->mc_promisc_last = table->mc_promisc; 1771 } 1772 1773 #ifdef CONFIG_RFS_ACCEL 1774 1775 bool efx_mcdi_filter_rfs_expire_one(struct efx_nic *efx, u32 flow_id, 1776 unsigned int filter_idx) 1777 { 1778 struct efx_filter_spec *spec, saved_spec; 1779 struct efx_mcdi_filter_table *table; 1780 struct efx_arfs_rule *rule = NULL; 1781 bool ret = true, force = false; 1782 u16 arfs_id; 1783 1784 down_read(&efx->filter_sem); 1785 table = efx->filter_state; 1786 down_write(&table->lock); 1787 spec = efx_mcdi_filter_entry_spec(table, filter_idx); 1788 1789 if (!spec || spec->priority != EFX_FILTER_PRI_HINT) 1790 goto out_unlock; 1791 1792 spin_lock_bh(&efx->rps_hash_lock); 1793 if (!efx->rps_hash_table) { 1794 /* In the absence of the table, we always return 0 to ARFS. */ 1795 arfs_id = 0; 1796 } else { 1797 rule = efx_rps_hash_find(efx, spec); 1798 if (!rule) 1799 /* ARFS table doesn't know of this filter, so remove it */ 1800 goto expire; 1801 arfs_id = rule->arfs_id; 1802 ret = efx_rps_check_rule(rule, filter_idx, &force); 1803 if (force) 1804 goto expire; 1805 if (!ret) { 1806 spin_unlock_bh(&efx->rps_hash_lock); 1807 goto out_unlock; 1808 } 1809 } 1810 if (!rps_may_expire_flow(efx->net_dev, spec->dmaq_id, flow_id, arfs_id)) 1811 ret = false; 1812 else if (rule) 1813 rule->filter_id = EFX_ARFS_FILTER_ID_REMOVING; 1814 expire: 1815 saved_spec = *spec; /* remove operation will kfree spec */ 1816 spin_unlock_bh(&efx->rps_hash_lock); 1817 /* 1818 * At this point (since we dropped the lock), another thread might queue 1819 * up a fresh insertion request (but the actual insertion will be held 1820 * up by our possession of the filter table lock). In that case, it 1821 * will set rule->filter_id to EFX_ARFS_FILTER_ID_PENDING, meaning that 1822 * the rule is not removed by efx_rps_hash_del() below. 1823 */ 1824 if (ret) 1825 ret = efx_mcdi_filter_remove_internal(efx, 1U << spec->priority, 1826 filter_idx, true) == 0; 1827 /* 1828 * While we can't safely dereference rule (we dropped the lock), we can 1829 * still test it for NULL. 1830 */ 1831 if (ret && rule) { 1832 /* Expiring, so remove entry from ARFS table */ 1833 spin_lock_bh(&efx->rps_hash_lock); 1834 efx_rps_hash_del(efx, &saved_spec); 1835 spin_unlock_bh(&efx->rps_hash_lock); 1836 } 1837 out_unlock: 1838 up_write(&table->lock); 1839 up_read(&efx->filter_sem); 1840 return ret; 1841 } 1842 1843 #endif /* CONFIG_RFS_ACCEL */ 1844 1845 #define RSS_MODE_HASH_ADDRS (1 << RSS_MODE_HASH_SRC_ADDR_LBN |\ 1846 1 << RSS_MODE_HASH_DST_ADDR_LBN) 1847 #define RSS_MODE_HASH_PORTS (1 << RSS_MODE_HASH_SRC_PORT_LBN |\ 1848 1 << RSS_MODE_HASH_DST_PORT_LBN) 1849 #define RSS_CONTEXT_FLAGS_DEFAULT (1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV4_EN_LBN |\ 1850 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV4_EN_LBN |\ 1851 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_IPV6_EN_LBN |\ 1852 1 << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TOEPLITZ_TCPV6_EN_LBN |\ 1853 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV4_RSS_MODE_LBN |\ 1854 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN |\ 1855 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV4_RSS_MODE_LBN |\ 1856 (RSS_MODE_HASH_ADDRS | RSS_MODE_HASH_PORTS) << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_TCP_IPV6_RSS_MODE_LBN |\ 1857 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN |\ 1858 RSS_MODE_HASH_ADDRS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_OTHER_IPV6_RSS_MODE_LBN) 1859 1860 int efx_mcdi_get_rss_context_flags(struct efx_nic *efx, u32 context, u32 *flags) 1861 { 1862 /* 1863 * Firmware had a bug (sfc bug 61952) where it would not actually 1864 * fill in the flags field in the response to MC_CMD_RSS_CONTEXT_GET_FLAGS. 1865 * This meant that it would always contain whatever was previously 1866 * in the MCDI buffer. Fortunately, all firmware versions with 1867 * this bug have the same default flags value for a newly-allocated 1868 * RSS context, and the only time we want to get the flags is just 1869 * after allocating. Moreover, the response has a 32-bit hole 1870 * where the context ID would be in the request, so we can use an 1871 * overlength buffer in the request and pre-fill the flags field 1872 * with what we believe the default to be. Thus if the firmware 1873 * has the bug, it will leave our pre-filled value in the flags 1874 * field of the response, and we will get the right answer. 1875 * 1876 * However, this does mean that this function should NOT be used if 1877 * the RSS context flags might not be their defaults - it is ONLY 1878 * reliably correct for a newly-allocated RSS context. 1879 */ 1880 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 1881 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN); 1882 size_t outlen; 1883 int rc; 1884 1885 /* Check we have a hole for the context ID */ 1886 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_FLAGS_IN_LEN != MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_FLAGS_OFST); 1887 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_IN_RSS_CONTEXT_ID, context); 1888 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS, 1889 RSS_CONTEXT_FLAGS_DEFAULT); 1890 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_FLAGS, inbuf, 1891 sizeof(inbuf), outbuf, sizeof(outbuf), &outlen); 1892 if (rc == 0) { 1893 if (outlen < MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_LEN) 1894 rc = -EIO; 1895 else 1896 *flags = MCDI_DWORD(outbuf, RSS_CONTEXT_GET_FLAGS_OUT_FLAGS); 1897 } 1898 return rc; 1899 } 1900 1901 /* 1902 * Attempt to enable 4-tuple UDP hashing on the specified RSS context. 1903 * If we fail, we just leave the RSS context at its default hash settings, 1904 * which is safe but may slightly reduce performance. 1905 * Defaults are 4-tuple for TCP and 2-tuple for UDP and other-IP, so we 1906 * just need to set the UDP ports flags (for both IP versions). 1907 */ 1908 void efx_mcdi_set_rss_context_flags(struct efx_nic *efx, 1909 struct efx_rss_context *ctx) 1910 { 1911 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN); 1912 u32 flags; 1913 1914 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN != 0); 1915 1916 if (efx_mcdi_get_rss_context_flags(efx, ctx->context_id, &flags) != 0) 1917 return; 1918 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 1919 ctx->context_id); 1920 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV4_RSS_MODE_LBN; 1921 flags |= RSS_MODE_HASH_PORTS << MC_CMD_RSS_CONTEXT_GET_FLAGS_OUT_UDP_IPV6_RSS_MODE_LBN; 1922 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, flags); 1923 if (!efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_FLAGS, inbuf, sizeof(inbuf), 1924 NULL, 0, NULL)) 1925 /* Succeeded, so UDP 4-tuple is now enabled */ 1926 ctx->rx_hash_udp_4tuple = true; 1927 } 1928 1929 static int efx_mcdi_filter_alloc_rss_context(struct efx_nic *efx, bool exclusive, 1930 struct efx_rss_context *ctx, 1931 unsigned *context_size) 1932 { 1933 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN); 1934 MCDI_DECLARE_BUF(outbuf, MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 1935 size_t outlen; 1936 int rc; 1937 u32 alloc_type = exclusive ? 1938 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE : 1939 MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 1940 unsigned rss_spread = exclusive ? 1941 efx->rss_spread : 1942 min(rounddown_pow_of_two(efx->rss_spread), 1943 EFX_EF10_MAX_SHARED_RSS_CONTEXT_SIZE); 1944 1945 if (!exclusive && rss_spread == 1) { 1946 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 1947 if (context_size) 1948 *context_size = 1; 1949 return 0; 1950 } 1951 1952 if (efx_has_cap(efx, RX_RSS_LIMITED)) 1953 return -EOPNOTSUPP; 1954 1955 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 1956 efx->vport_id); 1957 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_TYPE, alloc_type); 1958 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, rss_spread); 1959 1960 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_ALLOC, inbuf, sizeof(inbuf), 1961 outbuf, sizeof(outbuf), &outlen); 1962 if (rc != 0) 1963 return rc; 1964 1965 if (outlen < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) 1966 return -EIO; 1967 1968 ctx->context_id = MCDI_DWORD(outbuf, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 1969 1970 if (context_size) 1971 *context_size = rss_spread; 1972 1973 if (efx_has_cap(efx, ADDITIONAL_RSS_MODES)) 1974 efx_mcdi_set_rss_context_flags(efx, ctx); 1975 1976 return 0; 1977 } 1978 1979 static int efx_mcdi_filter_free_rss_context(struct efx_nic *efx, u32 context) 1980 { 1981 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_FREE_IN_LEN); 1982 1983 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, 1984 context); 1985 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_FREE, inbuf, sizeof(inbuf), 1986 NULL, 0, NULL); 1987 } 1988 1989 static int efx_mcdi_filter_populate_rss_table(struct efx_nic *efx, u32 context, 1990 const u32 *rx_indir_table, const u8 *key) 1991 { 1992 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN); 1993 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN); 1994 int i, rc; 1995 1996 MCDI_SET_DWORD(tablebuf, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 1997 context); 1998 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_indir_table) != 1999 MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN); 2000 2001 /* This iterates over the length of efx->rss_context.rx_indir_table, but 2002 * copies bytes from rx_indir_table. That's because the latter is a 2003 * pointer rather than an array, but should have the same length. 2004 * The efx->rss_context.rx_hash_key loop below is similar. 2005 */ 2006 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_indir_table); ++i) 2007 MCDI_PTR(tablebuf, 2008 RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE)[i] = 2009 (u8) rx_indir_table[i]; 2010 2011 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_TABLE, tablebuf, 2012 sizeof(tablebuf), NULL, 0, NULL); 2013 if (rc != 0) 2014 return rc; 2015 2016 MCDI_SET_DWORD(keybuf, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 2017 context); 2018 BUILD_BUG_ON(ARRAY_SIZE(efx->rss_context.rx_hash_key) != 2019 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2020 for (i = 0; i < ARRAY_SIZE(efx->rss_context.rx_hash_key); ++i) 2021 MCDI_PTR(keybuf, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY)[i] = key[i]; 2022 2023 return efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_SET_KEY, keybuf, 2024 sizeof(keybuf), NULL, 0, NULL); 2025 } 2026 2027 void efx_mcdi_rx_free_indir_table(struct efx_nic *efx) 2028 { 2029 int rc; 2030 2031 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) { 2032 rc = efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id); 2033 WARN_ON(rc != 0); 2034 } 2035 efx->rss_context.context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 2036 } 2037 2038 static int efx_mcdi_filter_rx_push_shared_rss_config(struct efx_nic *efx, 2039 unsigned *context_size) 2040 { 2041 struct efx_mcdi_filter_table *table = efx->filter_state; 2042 int rc = efx_mcdi_filter_alloc_rss_context(efx, false, &efx->rss_context, 2043 context_size); 2044 2045 if (rc != 0) 2046 return rc; 2047 2048 table->rx_rss_context_exclusive = false; 2049 efx_set_default_rx_indir_table(efx, &efx->rss_context); 2050 return 0; 2051 } 2052 2053 static int efx_mcdi_filter_rx_push_exclusive_rss_config(struct efx_nic *efx, 2054 const u32 *rx_indir_table, 2055 const u8 *key) 2056 { 2057 struct efx_mcdi_filter_table *table = efx->filter_state; 2058 u32 old_rx_rss_context = efx->rss_context.context_id; 2059 int rc; 2060 2061 if (efx->rss_context.context_id == EFX_MCDI_RSS_CONTEXT_INVALID || 2062 !table->rx_rss_context_exclusive) { 2063 rc = efx_mcdi_filter_alloc_rss_context(efx, true, &efx->rss_context, 2064 NULL); 2065 if (rc == -EOPNOTSUPP) 2066 return rc; 2067 else if (rc != 0) 2068 goto fail1; 2069 } 2070 2071 rc = efx_mcdi_filter_populate_rss_table(efx, efx->rss_context.context_id, 2072 rx_indir_table, key); 2073 if (rc != 0) 2074 goto fail2; 2075 2076 if (efx->rss_context.context_id != old_rx_rss_context && 2077 old_rx_rss_context != EFX_MCDI_RSS_CONTEXT_INVALID) 2078 WARN_ON(efx_mcdi_filter_free_rss_context(efx, old_rx_rss_context) != 0); 2079 table->rx_rss_context_exclusive = true; 2080 if (rx_indir_table != efx->rss_context.rx_indir_table) 2081 memcpy(efx->rss_context.rx_indir_table, rx_indir_table, 2082 sizeof(efx->rss_context.rx_indir_table)); 2083 if (key != efx->rss_context.rx_hash_key) 2084 memcpy(efx->rss_context.rx_hash_key, key, 2085 efx->type->rx_hash_key_size); 2086 2087 return 0; 2088 2089 fail2: 2090 if (old_rx_rss_context != efx->rss_context.context_id) { 2091 WARN_ON(efx_mcdi_filter_free_rss_context(efx, efx->rss_context.context_id) != 0); 2092 efx->rss_context.context_id = old_rx_rss_context; 2093 } 2094 fail1: 2095 netif_err(efx, hw, efx->net_dev, "%s: failed rc=%d\n", __func__, rc); 2096 return rc; 2097 } 2098 2099 int efx_mcdi_rx_push_rss_context_config(struct efx_nic *efx, 2100 struct efx_rss_context *ctx, 2101 const u32 *rx_indir_table, 2102 const u8 *key) 2103 { 2104 int rc; 2105 2106 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2107 2108 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) { 2109 rc = efx_mcdi_filter_alloc_rss_context(efx, true, ctx, NULL); 2110 if (rc) 2111 return rc; 2112 } 2113 2114 if (!rx_indir_table) /* Delete this context */ 2115 return efx_mcdi_filter_free_rss_context(efx, ctx->context_id); 2116 2117 rc = efx_mcdi_filter_populate_rss_table(efx, ctx->context_id, 2118 rx_indir_table, key); 2119 if (rc) 2120 return rc; 2121 2122 memcpy(ctx->rx_indir_table, rx_indir_table, 2123 sizeof(efx->rss_context.rx_indir_table)); 2124 memcpy(ctx->rx_hash_key, key, efx->type->rx_hash_key_size); 2125 2126 return 0; 2127 } 2128 2129 int efx_mcdi_rx_pull_rss_context_config(struct efx_nic *efx, 2130 struct efx_rss_context *ctx) 2131 { 2132 MCDI_DECLARE_BUF(inbuf, MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN); 2133 MCDI_DECLARE_BUF(tablebuf, MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN); 2134 MCDI_DECLARE_BUF(keybuf, MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN); 2135 size_t outlen; 2136 int rc, i; 2137 2138 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2139 2140 BUILD_BUG_ON(MC_CMD_RSS_CONTEXT_GET_TABLE_IN_LEN != 2141 MC_CMD_RSS_CONTEXT_GET_KEY_IN_LEN); 2142 2143 if (ctx->context_id == EFX_MCDI_RSS_CONTEXT_INVALID) 2144 return -ENOENT; 2145 2146 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_TABLE_IN_RSS_CONTEXT_ID, 2147 ctx->context_id); 2148 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_indir_table) != 2149 MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE_LEN); 2150 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_TABLE, inbuf, sizeof(inbuf), 2151 tablebuf, sizeof(tablebuf), &outlen); 2152 if (rc != 0) 2153 return rc; 2154 2155 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_TABLE_OUT_LEN)) 2156 return -EIO; 2157 2158 for (i = 0; i < ARRAY_SIZE(ctx->rx_indir_table); i++) 2159 ctx->rx_indir_table[i] = MCDI_PTR(tablebuf, 2160 RSS_CONTEXT_GET_TABLE_OUT_INDIRECTION_TABLE)[i]; 2161 2162 MCDI_SET_DWORD(inbuf, RSS_CONTEXT_GET_KEY_IN_RSS_CONTEXT_ID, 2163 ctx->context_id); 2164 BUILD_BUG_ON(ARRAY_SIZE(ctx->rx_hash_key) != 2165 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 2166 rc = efx_mcdi_rpc(efx, MC_CMD_RSS_CONTEXT_GET_KEY, inbuf, sizeof(inbuf), 2167 keybuf, sizeof(keybuf), &outlen); 2168 if (rc != 0) 2169 return rc; 2170 2171 if (WARN_ON(outlen != MC_CMD_RSS_CONTEXT_GET_KEY_OUT_LEN)) 2172 return -EIO; 2173 2174 for (i = 0; i < ARRAY_SIZE(ctx->rx_hash_key); ++i) 2175 ctx->rx_hash_key[i] = MCDI_PTR( 2176 keybuf, RSS_CONTEXT_GET_KEY_OUT_TOEPLITZ_KEY)[i]; 2177 2178 return 0; 2179 } 2180 2181 int efx_mcdi_rx_pull_rss_config(struct efx_nic *efx) 2182 { 2183 int rc; 2184 2185 mutex_lock(&efx->rss_lock); 2186 rc = efx_mcdi_rx_pull_rss_context_config(efx, &efx->rss_context); 2187 mutex_unlock(&efx->rss_lock); 2188 return rc; 2189 } 2190 2191 void efx_mcdi_rx_restore_rss_contexts(struct efx_nic *efx) 2192 { 2193 struct efx_mcdi_filter_table *table = efx->filter_state; 2194 struct efx_rss_context *ctx; 2195 int rc; 2196 2197 WARN_ON(!mutex_is_locked(&efx->rss_lock)); 2198 2199 if (!table->must_restore_rss_contexts) 2200 return; 2201 2202 list_for_each_entry(ctx, &efx->rss_context.list, list) { 2203 /* previous NIC RSS context is gone */ 2204 ctx->context_id = EFX_MCDI_RSS_CONTEXT_INVALID; 2205 /* so try to allocate a new one */ 2206 rc = efx_mcdi_rx_push_rss_context_config(efx, ctx, 2207 ctx->rx_indir_table, 2208 ctx->rx_hash_key); 2209 if (rc) 2210 netif_warn(efx, probe, efx->net_dev, 2211 "failed to restore RSS context %u, rc=%d" 2212 "; RSS filters may fail to be applied\n", 2213 ctx->user_id, rc); 2214 } 2215 table->must_restore_rss_contexts = false; 2216 } 2217 2218 int efx_mcdi_pf_rx_push_rss_config(struct efx_nic *efx, bool user, 2219 const u32 *rx_indir_table, 2220 const u8 *key) 2221 { 2222 int rc; 2223 2224 if (efx->rss_spread == 1) 2225 return 0; 2226 2227 if (!key) 2228 key = efx->rss_context.rx_hash_key; 2229 2230 rc = efx_mcdi_filter_rx_push_exclusive_rss_config(efx, rx_indir_table, key); 2231 2232 if (rc == -ENOBUFS && !user) { 2233 unsigned context_size; 2234 bool mismatch = false; 2235 size_t i; 2236 2237 for (i = 0; 2238 i < ARRAY_SIZE(efx->rss_context.rx_indir_table) && !mismatch; 2239 i++) 2240 mismatch = rx_indir_table[i] != 2241 ethtool_rxfh_indir_default(i, efx->rss_spread); 2242 2243 rc = efx_mcdi_filter_rx_push_shared_rss_config(efx, &context_size); 2244 if (rc == 0) { 2245 if (context_size != efx->rss_spread) 2246 netif_warn(efx, probe, efx->net_dev, 2247 "Could not allocate an exclusive RSS" 2248 " context; allocated a shared one of" 2249 " different size." 2250 " Wanted %u, got %u.\n", 2251 efx->rss_spread, context_size); 2252 else if (mismatch) 2253 netif_warn(efx, probe, efx->net_dev, 2254 "Could not allocate an exclusive RSS" 2255 " context; allocated a shared one but" 2256 " could not apply custom" 2257 " indirection.\n"); 2258 else 2259 netif_info(efx, probe, efx->net_dev, 2260 "Could not allocate an exclusive RSS" 2261 " context; allocated a shared one.\n"); 2262 } 2263 } 2264 return rc; 2265 } 2266 2267 int efx_mcdi_vf_rx_push_rss_config(struct efx_nic *efx, bool user, 2268 const u32 *rx_indir_table 2269 __attribute__ ((unused)), 2270 const u8 *key 2271 __attribute__ ((unused))) 2272 { 2273 if (user) 2274 return -EOPNOTSUPP; 2275 if (efx->rss_context.context_id != EFX_MCDI_RSS_CONTEXT_INVALID) 2276 return 0; 2277 return efx_mcdi_filter_rx_push_shared_rss_config(efx, NULL); 2278 } 2279 2280 int efx_mcdi_push_default_indir_table(struct efx_nic *efx, 2281 unsigned int rss_spread) 2282 { 2283 int rc = 0; 2284 2285 if (efx->rss_spread == rss_spread) 2286 return 0; 2287 2288 efx->rss_spread = rss_spread; 2289 if (!efx->filter_state) 2290 return 0; 2291 2292 efx_mcdi_rx_free_indir_table(efx); 2293 if (rss_spread > 1) { 2294 efx_set_default_rx_indir_table(efx, &efx->rss_context); 2295 rc = efx->type->rx_push_rss_config(efx, false, 2296 efx->rss_context.rx_indir_table, NULL); 2297 } 2298 return rc; 2299 } 2300