1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/mlx5/driver.h> 34 #include <linux/mlx5/fs.h> 35 #include <linux/rbtree.h> 36 #include "mlx5_core.h" 37 #include "fs_core.h" 38 #include "fs_cmd.h" 39 40 #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000) 41 /* Max number of counters to query in bulk read is 32K */ 42 #define MLX5_SW_MAX_COUNTERS_BULK BIT(15) 43 #define MLX5_SF_NUM_COUNTERS_BULK 8 44 #define MLX5_FC_POOL_MAX_THRESHOLD BIT(18) 45 #define MLX5_FC_POOL_USED_BUFF_RATIO 10 46 47 struct mlx5_fc_cache { 48 u64 packets; 49 u64 bytes; 50 u64 lastuse; 51 }; 52 53 struct mlx5_fc { 54 struct list_head list; 55 struct llist_node addlist; 56 struct llist_node dellist; 57 58 /* last{packets,bytes} members are used when calculating the delta since 59 * last reading 60 */ 61 u64 lastpackets; 62 u64 lastbytes; 63 64 struct mlx5_fc_bulk *bulk; 65 u32 id; 66 bool aging; 67 68 struct mlx5_fc_cache cache ____cacheline_aligned_in_smp; 69 }; 70 71 static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev); 72 static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool); 73 static struct mlx5_fc *mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool); 74 static void mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc); 75 76 /* locking scheme: 77 * 78 * It is the responsibility of the user to prevent concurrent calls or bad 79 * ordering to mlx5_fc_create(), mlx5_fc_destroy() and accessing a reference 80 * to struct mlx5_fc. 81 * e.g en_tc.c is protected by RTNL lock of its caller, and will never call a 82 * dump (access to struct mlx5_fc) after a counter is destroyed. 83 * 84 * access to counter list: 85 * - create (user context) 86 * - mlx5_fc_create() only adds to an addlist to be used by 87 * mlx5_fc_stats_work(). addlist is a lockless single linked list 88 * that doesn't require any additional synchronization when adding single 89 * node. 90 * - spawn thread to do the actual destroy 91 * 92 * - destroy (user context) 93 * - add a counter to lockless dellist 94 * - spawn thread to do the actual del 95 * 96 * - dump (user context) 97 * user should not call dump after destroy 98 * 99 * - query (single thread workqueue context) 100 * destroy/dump - no conflict (see destroy) 101 * query/dump - packets and bytes might be inconsistent (since update is not 102 * atomic) 103 * query/create - no conflict (see create) 104 * since every create/destroy spawn the work, only after necessary time has 105 * elapsed, the thread will actually query the hardware. 106 */ 107 108 static struct list_head *mlx5_fc_counters_lookup_next(struct mlx5_core_dev *dev, 109 u32 id) 110 { 111 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 112 unsigned long next_id = (unsigned long)id + 1; 113 struct mlx5_fc *counter; 114 unsigned long tmp; 115 116 rcu_read_lock(); 117 /* skip counters that are in idr, but not yet in counters list */ 118 idr_for_each_entry_continue_ul(&fc_stats->counters_idr, 119 counter, tmp, next_id) { 120 if (!list_empty(&counter->list)) 121 break; 122 } 123 rcu_read_unlock(); 124 125 return counter ? &counter->list : &fc_stats->counters; 126 } 127 128 static void mlx5_fc_stats_insert(struct mlx5_core_dev *dev, 129 struct mlx5_fc *counter) 130 { 131 struct list_head *next = mlx5_fc_counters_lookup_next(dev, counter->id); 132 133 list_add_tail(&counter->list, next); 134 } 135 136 static void mlx5_fc_stats_remove(struct mlx5_core_dev *dev, 137 struct mlx5_fc *counter) 138 { 139 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 140 141 list_del(&counter->list); 142 143 spin_lock(&fc_stats->counters_idr_lock); 144 WARN_ON(!idr_remove(&fc_stats->counters_idr, counter->id)); 145 spin_unlock(&fc_stats->counters_idr_lock); 146 } 147 148 static int get_max_bulk_query_len(struct mlx5_core_dev *dev) 149 { 150 int num_counters_bulk = mlx5_core_is_sf(dev) ? 151 MLX5_SF_NUM_COUNTERS_BULK : 152 MLX5_SW_MAX_COUNTERS_BULK; 153 154 return min_t(int, num_counters_bulk, 155 (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk))); 156 } 157 158 static void update_counter_cache(int index, u32 *bulk_raw_data, 159 struct mlx5_fc_cache *cache) 160 { 161 void *stats = MLX5_ADDR_OF(query_flow_counter_out, bulk_raw_data, 162 flow_statistics[index]); 163 u64 packets = MLX5_GET64(traffic_counter, stats, packets); 164 u64 bytes = MLX5_GET64(traffic_counter, stats, octets); 165 166 if (cache->packets == packets) 167 return; 168 169 cache->packets = packets; 170 cache->bytes = bytes; 171 cache->lastuse = jiffies; 172 } 173 174 static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, 175 struct mlx5_fc *first, 176 u32 last_id) 177 { 178 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 179 bool query_more_counters = (first->id <= last_id); 180 int max_bulk_len = get_max_bulk_query_len(dev); 181 u32 *data = fc_stats->bulk_query_out; 182 struct mlx5_fc *counter = first; 183 u32 bulk_base_id; 184 int bulk_len; 185 int err; 186 187 while (query_more_counters) { 188 /* first id must be aligned to 4 when using bulk query */ 189 bulk_base_id = counter->id & ~0x3; 190 191 /* number of counters to query inc. the last counter */ 192 bulk_len = min_t(int, max_bulk_len, 193 ALIGN(last_id - bulk_base_id + 1, 4)); 194 195 err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, 196 data); 197 if (err) { 198 mlx5_core_err(dev, "Error doing bulk query: %d\n", err); 199 return; 200 } 201 query_more_counters = false; 202 203 list_for_each_entry_from(counter, &fc_stats->counters, list) { 204 int counter_index = counter->id - bulk_base_id; 205 struct mlx5_fc_cache *cache = &counter->cache; 206 207 if (counter->id >= bulk_base_id + bulk_len) { 208 query_more_counters = true; 209 break; 210 } 211 212 update_counter_cache(counter_index, data, cache); 213 } 214 } 215 } 216 217 static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter) 218 { 219 mlx5_cmd_fc_free(dev, counter->id); 220 kfree(counter); 221 } 222 223 static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) 224 { 225 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 226 227 if (counter->bulk) 228 mlx5_fc_pool_release_counter(&fc_stats->fc_pool, counter); 229 else 230 mlx5_fc_free(dev, counter); 231 } 232 233 static void mlx5_fc_stats_work(struct work_struct *work) 234 { 235 struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev, 236 priv.fc_stats.work.work); 237 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 238 /* Take dellist first to ensure that counters cannot be deleted before 239 * they are inserted. 240 */ 241 struct llist_node *dellist = llist_del_all(&fc_stats->dellist); 242 struct llist_node *addlist = llist_del_all(&fc_stats->addlist); 243 struct mlx5_fc *counter = NULL, *last = NULL, *tmp; 244 unsigned long now = jiffies; 245 246 if (addlist || !list_empty(&fc_stats->counters)) 247 queue_delayed_work(fc_stats->wq, &fc_stats->work, 248 fc_stats->sampling_interval); 249 250 llist_for_each_entry(counter, addlist, addlist) 251 mlx5_fc_stats_insert(dev, counter); 252 253 llist_for_each_entry_safe(counter, tmp, dellist, dellist) { 254 mlx5_fc_stats_remove(dev, counter); 255 256 mlx5_fc_release(dev, counter); 257 } 258 259 if (time_before(now, fc_stats->next_query) || 260 list_empty(&fc_stats->counters)) 261 return; 262 last = list_last_entry(&fc_stats->counters, struct mlx5_fc, list); 263 264 counter = list_first_entry(&fc_stats->counters, struct mlx5_fc, 265 list); 266 if (counter) 267 mlx5_fc_stats_query_counter_range(dev, counter, last->id); 268 269 fc_stats->next_query = now + fc_stats->sampling_interval; 270 } 271 272 static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev) 273 { 274 struct mlx5_fc *counter; 275 int err; 276 277 counter = kzalloc(sizeof(*counter), GFP_KERNEL); 278 if (!counter) 279 return ERR_PTR(-ENOMEM); 280 281 err = mlx5_cmd_fc_alloc(dev, &counter->id); 282 if (err) { 283 kfree(counter); 284 return ERR_PTR(err); 285 } 286 287 return counter; 288 } 289 290 static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging) 291 { 292 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 293 struct mlx5_fc *counter; 294 295 if (aging && MLX5_CAP_GEN(dev, flow_counter_bulk_alloc) != 0) { 296 counter = mlx5_fc_pool_acquire_counter(&fc_stats->fc_pool); 297 if (!IS_ERR(counter)) 298 return counter; 299 } 300 301 return mlx5_fc_single_alloc(dev); 302 } 303 304 struct mlx5_fc *mlx5_fc_create_ex(struct mlx5_core_dev *dev, bool aging) 305 { 306 struct mlx5_fc *counter = mlx5_fc_acquire(dev, aging); 307 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 308 int err; 309 310 if (IS_ERR(counter)) 311 return counter; 312 313 INIT_LIST_HEAD(&counter->list); 314 counter->aging = aging; 315 316 if (aging) { 317 u32 id = counter->id; 318 319 counter->cache.lastuse = jiffies; 320 counter->lastbytes = counter->cache.bytes; 321 counter->lastpackets = counter->cache.packets; 322 323 idr_preload(GFP_KERNEL); 324 spin_lock(&fc_stats->counters_idr_lock); 325 326 err = idr_alloc_u32(&fc_stats->counters_idr, counter, &id, id, 327 GFP_NOWAIT); 328 329 spin_unlock(&fc_stats->counters_idr_lock); 330 idr_preload_end(); 331 if (err) 332 goto err_out_alloc; 333 334 llist_add(&counter->addlist, &fc_stats->addlist); 335 } 336 337 return counter; 338 339 err_out_alloc: 340 mlx5_fc_release(dev, counter); 341 return ERR_PTR(err); 342 } 343 344 struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging) 345 { 346 struct mlx5_fc *counter = mlx5_fc_create_ex(dev, aging); 347 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 348 349 if (aging) 350 mod_delayed_work(fc_stats->wq, &fc_stats->work, 0); 351 return counter; 352 } 353 EXPORT_SYMBOL(mlx5_fc_create); 354 355 u32 mlx5_fc_id(struct mlx5_fc *counter) 356 { 357 return counter->id; 358 } 359 EXPORT_SYMBOL(mlx5_fc_id); 360 361 void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter) 362 { 363 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 364 365 if (!counter) 366 return; 367 368 if (counter->aging) { 369 llist_add(&counter->dellist, &fc_stats->dellist); 370 mod_delayed_work(fc_stats->wq, &fc_stats->work, 0); 371 return; 372 } 373 374 mlx5_fc_release(dev, counter); 375 } 376 EXPORT_SYMBOL(mlx5_fc_destroy); 377 378 int mlx5_init_fc_stats(struct mlx5_core_dev *dev) 379 { 380 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 381 int max_bulk_len; 382 int max_out_len; 383 384 spin_lock_init(&fc_stats->counters_idr_lock); 385 idr_init(&fc_stats->counters_idr); 386 INIT_LIST_HEAD(&fc_stats->counters); 387 init_llist_head(&fc_stats->addlist); 388 init_llist_head(&fc_stats->dellist); 389 390 max_bulk_len = get_max_bulk_query_len(dev); 391 max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); 392 fc_stats->bulk_query_out = kzalloc(max_out_len, GFP_KERNEL); 393 if (!fc_stats->bulk_query_out) 394 return -ENOMEM; 395 396 fc_stats->wq = create_singlethread_workqueue("mlx5_fc"); 397 if (!fc_stats->wq) 398 goto err_wq_create; 399 400 fc_stats->sampling_interval = MLX5_FC_STATS_PERIOD; 401 INIT_DELAYED_WORK(&fc_stats->work, mlx5_fc_stats_work); 402 403 mlx5_fc_pool_init(&fc_stats->fc_pool, dev); 404 return 0; 405 406 err_wq_create: 407 kfree(fc_stats->bulk_query_out); 408 return -ENOMEM; 409 } 410 411 void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev) 412 { 413 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 414 struct llist_node *tmplist; 415 struct mlx5_fc *counter; 416 struct mlx5_fc *tmp; 417 418 cancel_delayed_work_sync(&dev->priv.fc_stats.work); 419 destroy_workqueue(dev->priv.fc_stats.wq); 420 dev->priv.fc_stats.wq = NULL; 421 422 tmplist = llist_del_all(&fc_stats->addlist); 423 llist_for_each_entry_safe(counter, tmp, tmplist, addlist) 424 mlx5_fc_release(dev, counter); 425 426 list_for_each_entry_safe(counter, tmp, &fc_stats->counters, list) 427 mlx5_fc_release(dev, counter); 428 429 mlx5_fc_pool_cleanup(&fc_stats->fc_pool); 430 idr_destroy(&fc_stats->counters_idr); 431 kfree(fc_stats->bulk_query_out); 432 } 433 434 int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter, 435 u64 *packets, u64 *bytes) 436 { 437 return mlx5_cmd_fc_query(dev, counter->id, packets, bytes); 438 } 439 EXPORT_SYMBOL(mlx5_fc_query); 440 441 u64 mlx5_fc_query_lastuse(struct mlx5_fc *counter) 442 { 443 return counter->cache.lastuse; 444 } 445 446 void mlx5_fc_query_cached(struct mlx5_fc *counter, 447 u64 *bytes, u64 *packets, u64 *lastuse) 448 { 449 struct mlx5_fc_cache c; 450 451 c = counter->cache; 452 453 *bytes = c.bytes - counter->lastbytes; 454 *packets = c.packets - counter->lastpackets; 455 *lastuse = c.lastuse; 456 457 counter->lastbytes = c.bytes; 458 counter->lastpackets = c.packets; 459 } 460 461 void mlx5_fc_queue_stats_work(struct mlx5_core_dev *dev, 462 struct delayed_work *dwork, 463 unsigned long delay) 464 { 465 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 466 467 queue_delayed_work(fc_stats->wq, dwork, delay); 468 } 469 470 void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev, 471 unsigned long interval) 472 { 473 struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; 474 475 fc_stats->sampling_interval = min_t(unsigned long, interval, 476 fc_stats->sampling_interval); 477 } 478 479 /* Flow counter bluks */ 480 481 struct mlx5_fc_bulk { 482 struct list_head pool_list; 483 u32 base_id; 484 int bulk_len; 485 unsigned long *bitmask; 486 struct mlx5_fc fcs[]; 487 }; 488 489 static void mlx5_fc_init(struct mlx5_fc *counter, struct mlx5_fc_bulk *bulk, 490 u32 id) 491 { 492 counter->bulk = bulk; 493 counter->id = id; 494 } 495 496 static int mlx5_fc_bulk_get_free_fcs_amount(struct mlx5_fc_bulk *bulk) 497 { 498 return bitmap_weight(bulk->bitmask, bulk->bulk_len); 499 } 500 501 static struct mlx5_fc_bulk *mlx5_fc_bulk_create(struct mlx5_core_dev *dev) 502 { 503 enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask; 504 struct mlx5_fc_bulk *bulk; 505 int err = -ENOMEM; 506 int bulk_len; 507 u32 base_id; 508 int i; 509 510 alloc_bitmask = MLX5_CAP_GEN(dev, flow_counter_bulk_alloc); 511 bulk_len = alloc_bitmask > 0 ? MLX5_FC_BULK_NUM_FCS(alloc_bitmask) : 1; 512 513 bulk = kvzalloc(struct_size(bulk, fcs, bulk_len), GFP_KERNEL); 514 if (!bulk) 515 goto err_alloc_bulk; 516 517 bulk->bitmask = kvcalloc(BITS_TO_LONGS(bulk_len), sizeof(unsigned long), 518 GFP_KERNEL); 519 if (!bulk->bitmask) 520 goto err_alloc_bitmask; 521 522 err = mlx5_cmd_fc_bulk_alloc(dev, alloc_bitmask, &base_id); 523 if (err) 524 goto err_mlx5_cmd_bulk_alloc; 525 526 bulk->base_id = base_id; 527 bulk->bulk_len = bulk_len; 528 for (i = 0; i < bulk_len; i++) { 529 mlx5_fc_init(&bulk->fcs[i], bulk, base_id + i); 530 set_bit(i, bulk->bitmask); 531 } 532 533 return bulk; 534 535 err_mlx5_cmd_bulk_alloc: 536 kvfree(bulk->bitmask); 537 err_alloc_bitmask: 538 kvfree(bulk); 539 err_alloc_bulk: 540 return ERR_PTR(err); 541 } 542 543 static int 544 mlx5_fc_bulk_destroy(struct mlx5_core_dev *dev, struct mlx5_fc_bulk *bulk) 545 { 546 if (mlx5_fc_bulk_get_free_fcs_amount(bulk) < bulk->bulk_len) { 547 mlx5_core_err(dev, "Freeing bulk before all counters were released\n"); 548 return -EBUSY; 549 } 550 551 mlx5_cmd_fc_free(dev, bulk->base_id); 552 kvfree(bulk->bitmask); 553 kvfree(bulk); 554 555 return 0; 556 } 557 558 static struct mlx5_fc *mlx5_fc_bulk_acquire_fc(struct mlx5_fc_bulk *bulk) 559 { 560 int free_fc_index = find_first_bit(bulk->bitmask, bulk->bulk_len); 561 562 if (free_fc_index >= bulk->bulk_len) 563 return ERR_PTR(-ENOSPC); 564 565 clear_bit(free_fc_index, bulk->bitmask); 566 return &bulk->fcs[free_fc_index]; 567 } 568 569 static int mlx5_fc_bulk_release_fc(struct mlx5_fc_bulk *bulk, struct mlx5_fc *fc) 570 { 571 int fc_index = fc->id - bulk->base_id; 572 573 if (test_bit(fc_index, bulk->bitmask)) 574 return -EINVAL; 575 576 set_bit(fc_index, bulk->bitmask); 577 return 0; 578 } 579 580 /* Flow counters pool API */ 581 582 static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev) 583 { 584 fc_pool->dev = dev; 585 mutex_init(&fc_pool->pool_lock); 586 INIT_LIST_HEAD(&fc_pool->fully_used); 587 INIT_LIST_HEAD(&fc_pool->partially_used); 588 INIT_LIST_HEAD(&fc_pool->unused); 589 fc_pool->available_fcs = 0; 590 fc_pool->used_fcs = 0; 591 fc_pool->threshold = 0; 592 } 593 594 static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool) 595 { 596 struct mlx5_core_dev *dev = fc_pool->dev; 597 struct mlx5_fc_bulk *bulk; 598 struct mlx5_fc_bulk *tmp; 599 600 list_for_each_entry_safe(bulk, tmp, &fc_pool->fully_used, pool_list) 601 mlx5_fc_bulk_destroy(dev, bulk); 602 list_for_each_entry_safe(bulk, tmp, &fc_pool->partially_used, pool_list) 603 mlx5_fc_bulk_destroy(dev, bulk); 604 list_for_each_entry_safe(bulk, tmp, &fc_pool->unused, pool_list) 605 mlx5_fc_bulk_destroy(dev, bulk); 606 } 607 608 static void mlx5_fc_pool_update_threshold(struct mlx5_fc_pool *fc_pool) 609 { 610 fc_pool->threshold = min_t(int, MLX5_FC_POOL_MAX_THRESHOLD, 611 fc_pool->used_fcs / MLX5_FC_POOL_USED_BUFF_RATIO); 612 } 613 614 static struct mlx5_fc_bulk * 615 mlx5_fc_pool_alloc_new_bulk(struct mlx5_fc_pool *fc_pool) 616 { 617 struct mlx5_core_dev *dev = fc_pool->dev; 618 struct mlx5_fc_bulk *new_bulk; 619 620 new_bulk = mlx5_fc_bulk_create(dev); 621 if (!IS_ERR(new_bulk)) 622 fc_pool->available_fcs += new_bulk->bulk_len; 623 mlx5_fc_pool_update_threshold(fc_pool); 624 return new_bulk; 625 } 626 627 static void 628 mlx5_fc_pool_free_bulk(struct mlx5_fc_pool *fc_pool, struct mlx5_fc_bulk *bulk) 629 { 630 struct mlx5_core_dev *dev = fc_pool->dev; 631 632 fc_pool->available_fcs -= bulk->bulk_len; 633 mlx5_fc_bulk_destroy(dev, bulk); 634 mlx5_fc_pool_update_threshold(fc_pool); 635 } 636 637 static struct mlx5_fc * 638 mlx5_fc_pool_acquire_from_list(struct list_head *src_list, 639 struct list_head *next_list, 640 bool move_non_full_bulk) 641 { 642 struct mlx5_fc_bulk *bulk; 643 struct mlx5_fc *fc; 644 645 if (list_empty(src_list)) 646 return ERR_PTR(-ENODATA); 647 648 bulk = list_first_entry(src_list, struct mlx5_fc_bulk, pool_list); 649 fc = mlx5_fc_bulk_acquire_fc(bulk); 650 if (move_non_full_bulk || mlx5_fc_bulk_get_free_fcs_amount(bulk) == 0) 651 list_move(&bulk->pool_list, next_list); 652 return fc; 653 } 654 655 static struct mlx5_fc * 656 mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool) 657 { 658 struct mlx5_fc_bulk *new_bulk; 659 struct mlx5_fc *fc; 660 661 mutex_lock(&fc_pool->pool_lock); 662 663 fc = mlx5_fc_pool_acquire_from_list(&fc_pool->partially_used, 664 &fc_pool->fully_used, false); 665 if (IS_ERR(fc)) 666 fc = mlx5_fc_pool_acquire_from_list(&fc_pool->unused, 667 &fc_pool->partially_used, 668 true); 669 if (IS_ERR(fc)) { 670 new_bulk = mlx5_fc_pool_alloc_new_bulk(fc_pool); 671 if (IS_ERR(new_bulk)) { 672 fc = ERR_CAST(new_bulk); 673 goto out; 674 } 675 fc = mlx5_fc_bulk_acquire_fc(new_bulk); 676 list_add(&new_bulk->pool_list, &fc_pool->partially_used); 677 } 678 fc_pool->available_fcs--; 679 fc_pool->used_fcs++; 680 681 out: 682 mutex_unlock(&fc_pool->pool_lock); 683 return fc; 684 } 685 686 static void 687 mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc) 688 { 689 struct mlx5_core_dev *dev = fc_pool->dev; 690 struct mlx5_fc_bulk *bulk = fc->bulk; 691 int bulk_free_fcs_amount; 692 693 mutex_lock(&fc_pool->pool_lock); 694 695 if (mlx5_fc_bulk_release_fc(bulk, fc)) { 696 mlx5_core_warn(dev, "Attempted to release a counter which is not acquired\n"); 697 goto unlock; 698 } 699 700 fc_pool->available_fcs++; 701 fc_pool->used_fcs--; 702 703 bulk_free_fcs_amount = mlx5_fc_bulk_get_free_fcs_amount(bulk); 704 if (bulk_free_fcs_amount == 1) 705 list_move_tail(&bulk->pool_list, &fc_pool->partially_used); 706 if (bulk_free_fcs_amount == bulk->bulk_len) { 707 list_del(&bulk->pool_list); 708 if (fc_pool->available_fcs > fc_pool->threshold) 709 mlx5_fc_pool_free_bulk(fc_pool, bulk); 710 else 711 list_add(&bulk->pool_list, &fc_pool->unused); 712 } 713 714 unlock: 715 mutex_unlock(&fc_pool->pool_lock); 716 } 717