1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2022 Advanced Micro Devices, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include "tc_counters.h" 12 #include "mae_counter_format.h" 13 #include "mae.h" 14 #include "rx_common.h" 15 16 /* Counter-management hashtables */ 17 18 static const struct rhashtable_params efx_tc_counter_id_ht_params = { 19 .key_len = offsetof(struct efx_tc_counter_index, linkage), 20 .key_offset = 0, 21 .head_offset = offsetof(struct efx_tc_counter_index, linkage), 22 }; 23 24 static const struct rhashtable_params efx_tc_counter_ht_params = { 25 .key_len = offsetof(struct efx_tc_counter, linkage), 26 .key_offset = 0, 27 .head_offset = offsetof(struct efx_tc_counter, linkage), 28 }; 29 30 static void efx_tc_counter_free(void *ptr, void *__unused) 31 { 32 struct efx_tc_counter *cnt = ptr; 33 34 kfree(cnt); 35 } 36 37 static void efx_tc_counter_id_free(void *ptr, void *__unused) 38 { 39 struct efx_tc_counter_index *ctr = ptr; 40 41 WARN_ON(refcount_read(&ctr->ref)); 42 kfree(ctr); 43 } 44 45 int efx_tc_init_counters(struct efx_nic *efx) 46 { 47 int rc; 48 49 rc = rhashtable_init(&efx->tc->counter_id_ht, &efx_tc_counter_id_ht_params); 50 if (rc < 0) 51 goto fail_counter_id_ht; 52 rc = rhashtable_init(&efx->tc->counter_ht, &efx_tc_counter_ht_params); 53 if (rc < 0) 54 goto fail_counter_ht; 55 return 0; 56 fail_counter_ht: 57 rhashtable_destroy(&efx->tc->counter_id_ht); 58 fail_counter_id_ht: 59 return rc; 60 } 61 62 /* Only call this in init failure teardown. 63 * Normal exit should fini instead as there may be entries in the table. 64 */ 65 void efx_tc_destroy_counters(struct efx_nic *efx) 66 { 67 rhashtable_destroy(&efx->tc->counter_ht); 68 rhashtable_destroy(&efx->tc->counter_id_ht); 69 } 70 71 void efx_tc_fini_counters(struct efx_nic *efx) 72 { 73 rhashtable_free_and_destroy(&efx->tc->counter_id_ht, efx_tc_counter_id_free, NULL); 74 rhashtable_free_and_destroy(&efx->tc->counter_ht, efx_tc_counter_free, NULL); 75 } 76 77 /* Counter allocation */ 78 79 static struct efx_tc_counter *efx_tc_flower_allocate_counter(struct efx_nic *efx, 80 int type) 81 { 82 struct efx_tc_counter *cnt; 83 int rc, rc2; 84 85 cnt = kzalloc(sizeof(*cnt), GFP_USER); 86 if (!cnt) 87 return ERR_PTR(-ENOMEM); 88 89 spin_lock_init(&cnt->lock); 90 cnt->touched = jiffies; 91 cnt->type = type; 92 93 rc = efx_mae_allocate_counter(efx, cnt); 94 if (rc) 95 goto fail1; 96 rc = rhashtable_insert_fast(&efx->tc->counter_ht, &cnt->linkage, 97 efx_tc_counter_ht_params); 98 if (rc) 99 goto fail2; 100 return cnt; 101 fail2: 102 /* If we get here, it implies that we couldn't insert into the table, 103 * which in turn probably means that the fw_id was already taken. 104 * In that case, it's unclear whether we really 'own' the fw_id; but 105 * the firmware seemed to think we did, so it's proper to free it. 106 */ 107 rc2 = efx_mae_free_counter(efx, cnt); 108 if (rc2) 109 netif_warn(efx, hw, efx->net_dev, 110 "Failed to free MAE counter %u, rc %d\n", 111 cnt->fw_id, rc2); 112 fail1: 113 kfree(cnt); 114 return ERR_PTR(rc > 0 ? -EIO : rc); 115 } 116 117 static void efx_tc_flower_release_counter(struct efx_nic *efx, 118 struct efx_tc_counter *cnt) 119 { 120 int rc; 121 122 rhashtable_remove_fast(&efx->tc->counter_ht, &cnt->linkage, 123 efx_tc_counter_ht_params); 124 rc = efx_mae_free_counter(efx, cnt); 125 if (rc) 126 netif_warn(efx, hw, efx->net_dev, 127 "Failed to free MAE counter %u, rc %d\n", 128 cnt->fw_id, rc); 129 /* This doesn't protect counter updates coming in arbitrarily long 130 * after we deleted the counter. The RCU just ensures that we won't 131 * free the counter while another thread has a pointer to it. 132 * Ensuring we don't update the wrong counter if the ID gets re-used 133 * is handled by the generation count. 134 */ 135 synchronize_rcu(); 136 EFX_WARN_ON_PARANOID(spin_is_locked(&cnt->lock)); 137 kfree(cnt); 138 } 139 140 static struct efx_tc_counter *efx_tc_flower_find_counter_by_fw_id( 141 struct efx_nic *efx, int type, u32 fw_id) 142 { 143 struct efx_tc_counter key = {}; 144 145 key.fw_id = fw_id; 146 key.type = type; 147 148 return rhashtable_lookup_fast(&efx->tc->counter_ht, &key, 149 efx_tc_counter_ht_params); 150 } 151 152 /* TC cookie to counter mapping */ 153 154 void efx_tc_flower_put_counter_index(struct efx_nic *efx, 155 struct efx_tc_counter_index *ctr) 156 { 157 if (!refcount_dec_and_test(&ctr->ref)) 158 return; /* still in use */ 159 rhashtable_remove_fast(&efx->tc->counter_id_ht, &ctr->linkage, 160 efx_tc_counter_id_ht_params); 161 efx_tc_flower_release_counter(efx, ctr->cnt); 162 kfree(ctr); 163 } 164 165 struct efx_tc_counter_index *efx_tc_flower_get_counter_index( 166 struct efx_nic *efx, unsigned long cookie, 167 enum efx_tc_counter_type type) 168 { 169 struct efx_tc_counter_index *ctr, *old; 170 struct efx_tc_counter *cnt; 171 172 ctr = kzalloc(sizeof(*ctr), GFP_USER); 173 if (!ctr) 174 return ERR_PTR(-ENOMEM); 175 ctr->cookie = cookie; 176 old = rhashtable_lookup_get_insert_fast(&efx->tc->counter_id_ht, 177 &ctr->linkage, 178 efx_tc_counter_id_ht_params); 179 if (old) { 180 /* don't need our new entry */ 181 kfree(ctr); 182 if (!refcount_inc_not_zero(&old->ref)) 183 return ERR_PTR(-EAGAIN); 184 /* existing entry found */ 185 ctr = old; 186 } else { 187 cnt = efx_tc_flower_allocate_counter(efx, type); 188 if (IS_ERR(cnt)) { 189 rhashtable_remove_fast(&efx->tc->counter_id_ht, 190 &ctr->linkage, 191 efx_tc_counter_id_ht_params); 192 kfree(ctr); 193 return (void *)cnt; /* it's an ERR_PTR */ 194 } 195 ctr->cnt = cnt; 196 refcount_set(&ctr->ref, 1); 197 } 198 return ctr; 199 } 200 201 struct efx_tc_counter_index *efx_tc_flower_find_counter_index( 202 struct efx_nic *efx, unsigned long cookie) 203 { 204 struct efx_tc_counter_index key = {}; 205 206 key.cookie = cookie; 207 return rhashtable_lookup_fast(&efx->tc->counter_id_ht, &key, 208 efx_tc_counter_id_ht_params); 209 } 210 211 /* TC Channel. Counter updates are delivered on this channel's RXQ. */ 212 213 static void efx_tc_handle_no_channel(struct efx_nic *efx) 214 { 215 netif_warn(efx, drv, efx->net_dev, 216 "MAE counters require MSI-X and 1 additional interrupt vector.\n"); 217 } 218 219 static int efx_tc_probe_channel(struct efx_channel *channel) 220 { 221 struct efx_rx_queue *rx_queue = &channel->rx_queue; 222 223 channel->irq_moderation_us = 0; 224 rx_queue->core_index = 0; 225 226 INIT_WORK(&rx_queue->grant_work, efx_mae_counters_grant_credits); 227 228 return 0; 229 } 230 231 static int efx_tc_start_channel(struct efx_channel *channel) 232 { 233 struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); 234 struct efx_nic *efx = channel->efx; 235 236 return efx_mae_start_counters(efx, rx_queue); 237 } 238 239 static void efx_tc_stop_channel(struct efx_channel *channel) 240 { 241 struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel); 242 struct efx_nic *efx = channel->efx; 243 int rc; 244 245 rc = efx_mae_stop_counters(efx, rx_queue); 246 if (rc) 247 netif_warn(efx, drv, efx->net_dev, 248 "Failed to stop MAE counters streaming, rc=%d.\n", 249 rc); 250 rx_queue->grant_credits = false; 251 flush_work(&rx_queue->grant_work); 252 } 253 254 static void efx_tc_remove_channel(struct efx_channel *channel) 255 { 256 } 257 258 static void efx_tc_get_channel_name(struct efx_channel *channel, 259 char *buf, size_t len) 260 { 261 snprintf(buf, len, "%s-mae", channel->efx->name); 262 } 263 264 static void efx_tc_counter_update(struct efx_nic *efx, 265 enum efx_tc_counter_type counter_type, 266 u32 counter_idx, u64 packets, u64 bytes, 267 u32 mark) 268 { 269 struct efx_tc_counter *cnt; 270 271 rcu_read_lock(); /* Protect against deletion of 'cnt' */ 272 cnt = efx_tc_flower_find_counter_by_fw_id(efx, counter_type, counter_idx); 273 if (!cnt) { 274 /* This can legitimately happen when a counter is removed, 275 * with updates for the counter still in-flight; however this 276 * should be an infrequent occurrence. 277 */ 278 if (net_ratelimit()) 279 netif_dbg(efx, drv, efx->net_dev, 280 "Got update for unwanted MAE counter %u type %u\n", 281 counter_idx, counter_type); 282 goto out; 283 } 284 285 spin_lock_bh(&cnt->lock); 286 if ((s32)mark - (s32)cnt->gen < 0) { 287 /* This counter update packet is from before the counter was 288 * allocated; thus it must be for a previous counter with 289 * the same ID that has since been freed, and it should be 290 * ignored. 291 */ 292 } else { 293 /* Update latest seen generation count. This ensures that 294 * even a long-lived counter won't start getting ignored if 295 * the generation count wraps around, unless it somehow 296 * manages to go 1<<31 generations without an update. 297 */ 298 cnt->gen = mark; 299 /* update counter values */ 300 cnt->packets += packets; 301 cnt->bytes += bytes; 302 cnt->touched = jiffies; 303 } 304 spin_unlock_bh(&cnt->lock); 305 out: 306 rcu_read_unlock(); 307 } 308 309 static void efx_tc_rx_version_1(struct efx_nic *efx, const u8 *data, u32 mark) 310 { 311 u16 n_counters, i; 312 313 /* Header format: 314 * + | 0 | 1 | 2 | 3 | 315 * 0 |version | reserved | 316 * 4 | seq_index | n_counters | 317 */ 318 319 n_counters = le16_to_cpu(*(const __le16 *)(data + 6)); 320 321 /* Counter update entry format: 322 * | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f | 323 * | counter_idx | packet_count | byte_count | 324 */ 325 for (i = 0; i < n_counters; i++) { 326 const void *entry = data + 8 + 16 * i; 327 u64 packet_count, byte_count; 328 u32 counter_idx; 329 330 counter_idx = le32_to_cpu(*(const __le32 *)entry); 331 packet_count = le32_to_cpu(*(const __le32 *)(entry + 4)) | 332 ((u64)le16_to_cpu(*(const __le16 *)(entry + 8)) << 32); 333 byte_count = le16_to_cpu(*(const __le16 *)(entry + 10)) | 334 ((u64)le32_to_cpu(*(const __le32 *)(entry + 12)) << 16); 335 efx_tc_counter_update(efx, EFX_TC_COUNTER_TYPE_AR, counter_idx, 336 packet_count, byte_count, mark); 337 } 338 } 339 340 #define TCV2_HDR_PTR(pkt, field) \ 341 ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 7), \ 342 (pkt) + ERF_SC_PACKETISER_HEADER_##field##_LBN / 8) 343 #define TCV2_HDR_BYTE(pkt, field) \ 344 ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 8),\ 345 *TCV2_HDR_PTR(pkt, field)) 346 #define TCV2_HDR_WORD(pkt, field) \ 347 ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_WIDTH != 16),\ 348 (void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_HEADER_##field##_LBN & 15), \ 349 *(__force const __le16 *)TCV2_HDR_PTR(pkt, field)) 350 #define TCV2_PKT_PTR(pkt, poff, i, field) \ 351 ((void)BUILD_BUG_ON_ZERO(ERF_SC_PACKETISER_PAYLOAD_##field##_LBN & 7), \ 352 (pkt) + ERF_SC_PACKETISER_PAYLOAD_##field##_LBN/8 + poff + \ 353 i * ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE) 354 355 /* Read a little-endian 48-bit field with 16-bit alignment */ 356 static u64 efx_tc_read48(const __le16 *field) 357 { 358 u64 out = 0; 359 int i; 360 361 for (i = 0; i < 3; i++) 362 out |= (u64)le16_to_cpu(field[i]) << (i * 16); 363 return out; 364 } 365 366 static enum efx_tc_counter_type efx_tc_rx_version_2(struct efx_nic *efx, 367 const u8 *data, u32 mark) 368 { 369 u8 payload_offset, header_offset, ident; 370 enum efx_tc_counter_type type; 371 u16 n_counters, i; 372 373 ident = TCV2_HDR_BYTE(data, IDENTIFIER); 374 switch (ident) { 375 case ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR: 376 type = EFX_TC_COUNTER_TYPE_AR; 377 break; 378 case ERF_SC_PACKETISER_HEADER_IDENTIFIER_CT: 379 type = EFX_TC_COUNTER_TYPE_CT; 380 break; 381 case ERF_SC_PACKETISER_HEADER_IDENTIFIER_OR: 382 type = EFX_TC_COUNTER_TYPE_OR; 383 break; 384 default: 385 if (net_ratelimit()) 386 netif_err(efx, drv, efx->net_dev, 387 "ignored v2 MAE counter packet (bad identifier %u" 388 "), counters may be inaccurate\n", ident); 389 return EFX_TC_COUNTER_TYPE_MAX; 390 } 391 header_offset = TCV2_HDR_BYTE(data, HEADER_OFFSET); 392 /* mae_counter_format.h implies that this offset is fixed, since it 393 * carries on with SOP-based LBNs for the fields in this header 394 */ 395 if (header_offset != ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT) { 396 if (net_ratelimit()) 397 netif_err(efx, drv, efx->net_dev, 398 "choked on v2 MAE counter packet (bad header_offset %u" 399 "), counters may be inaccurate\n", header_offset); 400 return EFX_TC_COUNTER_TYPE_MAX; 401 } 402 payload_offset = TCV2_HDR_BYTE(data, PAYLOAD_OFFSET); 403 n_counters = le16_to_cpu(TCV2_HDR_WORD(data, COUNT)); 404 405 for (i = 0; i < n_counters; i++) { 406 const void *counter_idx_p, *packet_count_p, *byte_count_p; 407 u64 packet_count, byte_count; 408 u32 counter_idx; 409 410 /* 24-bit field with 32-bit alignment */ 411 counter_idx_p = TCV2_PKT_PTR(data, payload_offset, i, COUNTER_INDEX); 412 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_WIDTH != 24); 413 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX_LBN & 31); 414 counter_idx = le32_to_cpu(*(const __le32 *)counter_idx_p) & 0xffffff; 415 /* 48-bit field with 16-bit alignment */ 416 packet_count_p = TCV2_PKT_PTR(data, payload_offset, i, PACKET_COUNT); 417 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_WIDTH != 48); 418 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LBN & 15); 419 packet_count = efx_tc_read48((const __le16 *)packet_count_p); 420 /* 48-bit field with 16-bit alignment */ 421 byte_count_p = TCV2_PKT_PTR(data, payload_offset, i, BYTE_COUNT); 422 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_WIDTH != 48); 423 BUILD_BUG_ON(ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LBN & 15); 424 byte_count = efx_tc_read48((const __le16 *)byte_count_p); 425 426 if (type == EFX_TC_COUNTER_TYPE_CT) { 427 /* CT counters are 1-bit saturating counters to update 428 * the lastuse time in CT stats. A received CT counter 429 * should have packet counter to 0 and only LSB bit on 430 * in byte counter. 431 */ 432 if (packet_count || byte_count != 1) 433 netdev_warn_once(efx->net_dev, 434 "CT counter with inconsistent state (%llu, %llu)\n", 435 packet_count, byte_count); 436 /* Do not increment the driver's byte counter */ 437 byte_count = 0; 438 } 439 440 efx_tc_counter_update(efx, type, counter_idx, packet_count, 441 byte_count, mark); 442 } 443 return type; 444 } 445 446 /* We always swallow the packet, whether successful or not, since it's not 447 * a network packet and shouldn't ever be forwarded to the stack. 448 * @mark is the generation count for counter allocations. 449 */ 450 static bool efx_tc_rx(struct efx_rx_queue *rx_queue, u32 mark) 451 { 452 struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 453 struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue, 454 channel->rx_pkt_index); 455 const u8 *data = efx_rx_buf_va(rx_buf); 456 struct efx_nic *efx = rx_queue->efx; 457 enum efx_tc_counter_type type; 458 u8 version; 459 460 /* version is always first byte of packet */ 461 version = *data; 462 switch (version) { 463 case 1: 464 type = EFX_TC_COUNTER_TYPE_AR; 465 efx_tc_rx_version_1(efx, data, mark); 466 break; 467 case ERF_SC_PACKETISER_HEADER_VERSION_VALUE: // 2 468 type = efx_tc_rx_version_2(efx, data, mark); 469 break; 470 default: 471 if (net_ratelimit()) 472 netif_err(efx, drv, efx->net_dev, 473 "choked on MAE counter packet (bad version %u" 474 "); counters may be inaccurate\n", 475 version); 476 goto out; 477 } 478 479 if (type < EFX_TC_COUNTER_TYPE_MAX) { 480 /* Update seen_gen unconditionally, to avoid a missed wakeup if 481 * we race with efx_mae_stop_counters(). 482 */ 483 efx->tc->seen_gen[type] = mark; 484 if (efx->tc->flush_counters && 485 (s32)(efx->tc->flush_gen[type] - mark) <= 0) 486 wake_up(&efx->tc->flush_wq); 487 } 488 out: 489 efx_free_rx_buffers(rx_queue, rx_buf, 1); 490 channel->rx_pkt_n_frags = 0; 491 return true; 492 } 493 494 const struct efx_channel_type efx_tc_channel_type = { 495 .handle_no_channel = efx_tc_handle_no_channel, 496 .pre_probe = efx_tc_probe_channel, 497 .start = efx_tc_start_channel, 498 .stop = efx_tc_stop_channel, 499 .post_remove = efx_tc_remove_channel, 500 .get_name = efx_tc_get_channel_name, 501 .receive_raw = efx_tc_rx, 502 .keep_eventq = true, 503 }; 504