1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 4 #include <linux/math64.h> 5 #include "lib/aso.h" 6 #include "en/tc/post_act.h" 7 #include "meter.h" 8 #include "en/tc_priv.h" 9 10 #define MLX5_START_COLOR_SHIFT 28 11 #define MLX5_METER_MODE_SHIFT 24 12 #define MLX5_CBS_EXP_SHIFT 24 13 #define MLX5_CBS_MAN_SHIFT 16 14 #define MLX5_CIR_EXP_SHIFT 8 15 16 /* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */ 17 #define MLX5_CONST_CIR 8000000000ULL 18 #define MLX5_CALC_CIR(m, e) ((MLX5_CONST_CIR * (m)) >> (e)) 19 #define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1) 20 21 /* cbs = cbs_mantissa*2^cbs_exponent */ 22 #define MLX5_CALC_CBS(m, e) ((m) << (e)) 23 #define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1) 24 #define MLX5_MAX_HW_CBS 0x7FFFFFFF 25 26 struct mlx5e_flow_meter_aso_obj { 27 struct list_head entry; 28 int base_id; 29 int total_meters; 30 31 unsigned long meters_map[0]; /* must be at the end of this struct */ 32 }; 33 34 struct mlx5e_flow_meters { 35 enum mlx5_flow_namespace_type ns_type; 36 struct mlx5_aso *aso; 37 struct mutex aso_lock; /* Protects aso operations */ 38 int log_granularity; 39 u32 pdn; 40 41 DECLARE_HASHTABLE(hashtbl, 8); 42 43 struct mutex sync_lock; /* protect flow meter operations */ 44 struct list_head partial_list; 45 struct list_head full_list; 46 47 struct mlx5_core_dev *mdev; 48 struct mlx5e_post_act *post_act; 49 }; 50 51 static void 52 mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp) 53 { 54 s64 _cir, _delta, delta = S64_MAX; 55 u8 e, _man = 0, _exp = 0; 56 u64 m; 57 58 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 59 m = cir << e; 60 if ((s64)m < 0) /* overflow */ 61 break; 62 m = div64_u64(m, MLX5_CONST_CIR); 63 if (m > 0xFF) /* man width 8 bit */ 64 continue; 65 _cir = MLX5_CALC_CIR(m, e); 66 _delta = cir - _cir; 67 if (_delta < delta) { 68 _man = m; 69 _exp = e; 70 if (!_delta) 71 goto found; 72 delta = _delta; 73 } 74 } 75 76 found: 77 *man = _man; 78 *exp = _exp; 79 } 80 81 static void 82 mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp) 83 { 84 s64 _cbs, _delta, delta = S64_MAX; 85 u8 e, _man = 0, _exp = 0; 86 u64 m; 87 88 for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ 89 m = cbs >> e; 90 if (m > 0xFF) /* man width 8 bit */ 91 continue; 92 _cbs = MLX5_CALC_CBS(m, e); 93 _delta = cbs - _cbs; 94 if (_delta < delta) { 95 _man = m; 96 _exp = e; 97 if (!_delta) 98 goto found; 99 delta = _delta; 100 } 101 } 102 103 found: 104 *man = _man; 105 *exp = _exp; 106 } 107 108 int 109 mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, 110 struct mlx5e_flow_meter_handle *meter, 111 struct mlx5e_flow_meter_params *meter_params) 112 { 113 struct mlx5_wqe_aso_ctrl_seg *aso_ctrl; 114 struct mlx5_wqe_aso_data_seg *aso_data; 115 struct mlx5e_flow_meters *flow_meters; 116 u8 cir_man, cir_exp, cbs_man, cbs_exp; 117 struct mlx5_aso_wqe *aso_wqe; 118 unsigned long expires; 119 struct mlx5_aso *aso; 120 u64 rate, burst; 121 u8 ds_cnt; 122 int err; 123 124 rate = meter_params->rate; 125 burst = meter_params->burst; 126 127 /* HW treats each packet as 128 bytes in PPS mode */ 128 if (meter_params->mode == MLX5_RATE_LIMIT_PPS) { 129 rate <<= 10; 130 burst <<= 7; 131 } 132 133 if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS) 134 return -EINVAL; 135 136 /* HW has limitation of total 31 bits for cbs */ 137 if (burst > MLX5_MAX_HW_CBS) { 138 mlx5_core_warn(mdev, 139 "burst(%lld) is too large, use HW allowed value(%d)\n", 140 burst, MLX5_MAX_HW_CBS); 141 burst = MLX5_MAX_HW_CBS; 142 } 143 144 mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode); 145 mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp); 146 mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n", 147 rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man); 148 mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp); 149 mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n", 150 burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man); 151 152 if (!cir_man || !cbs_man) 153 return -EINVAL; 154 155 flow_meters = meter->flow_meters; 156 aso = flow_meters->aso; 157 158 mutex_lock(&flow_meters->aso_lock); 159 aso_wqe = mlx5_aso_get_wqe(aso); 160 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS); 161 mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id, 162 MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); 163 164 aso_ctrl = &aso_wqe->aso_ctrl; 165 memset(aso_ctrl, 0, sizeof(*aso_ctrl)); 166 aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; 167 aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | 168 MLX5_ASO_ALWAYS_TRUE << 4; 169 aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6; 170 aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32)); 171 172 aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1); 173 memset(aso_data, 0, sizeof(*aso_data)); 174 aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */ 175 (MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT)); 176 if (meter_params->mode == MLX5_RATE_LIMIT_PPS) 177 aso_data->bytewise_data[meter->idx * 8] |= 178 cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT); 179 else 180 aso_data->bytewise_data[meter->idx * 8] |= 181 cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT); 182 183 aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) | 184 (cbs_man << MLX5_CBS_MAN_SHIFT) | 185 (cir_exp << MLX5_CIR_EXP_SHIFT) | 186 cir_man); 187 188 mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); 189 190 /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ 191 expires = jiffies + msecs_to_jiffies(10); 192 do { 193 err = mlx5_aso_poll_cq(aso, true); 194 if (err) 195 usleep_range(2, 10); 196 } while (err && time_is_after_jiffies(expires)); 197 mutex_unlock(&flow_meters->aso_lock); 198 199 return err; 200 } 201 202 static int 203 mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id) 204 { 205 u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; 206 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 207 struct mlx5_core_dev *mdev = flow_meters->mdev; 208 void *obj; 209 int err; 210 211 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); 212 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 213 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); 214 MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); 215 216 obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); 217 MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); 218 219 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 220 if (!err) { 221 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); 222 mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id); 223 } 224 225 return err; 226 } 227 228 static void 229 mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id) 230 { 231 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; 232 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; 233 234 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); 235 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, 236 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); 237 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); 238 239 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); 240 mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id); 241 } 242 243 static struct mlx5e_flow_meter_handle * 244 __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) 245 { 246 struct mlx5_core_dev *mdev = flow_meters->mdev; 247 struct mlx5e_flow_meter_aso_obj *meters_obj; 248 struct mlx5e_flow_meter_handle *meter; 249 struct mlx5_fc *counter; 250 int err, pos, total; 251 u32 id; 252 253 meter = kzalloc(sizeof(*meter), GFP_KERNEL); 254 if (!meter) 255 return ERR_PTR(-ENOMEM); 256 257 counter = mlx5_fc_create(mdev, true); 258 if (IS_ERR(counter)) { 259 err = PTR_ERR(counter); 260 goto err_red_counter; 261 } 262 meter->red_counter = counter; 263 264 counter = mlx5_fc_create(mdev, true); 265 if (IS_ERR(counter)) { 266 err = PTR_ERR(counter); 267 goto err_green_counter; 268 } 269 meter->green_counter = counter; 270 271 meters_obj = list_first_entry_or_null(&flow_meters->partial_list, 272 struct mlx5e_flow_meter_aso_obj, 273 entry); 274 /* 2 meters in one object */ 275 total = 1 << (flow_meters->log_granularity + 1); 276 if (!meters_obj) { 277 err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id); 278 if (err) { 279 mlx5_core_err(mdev, "Failed to create flow meter ASO object\n"); 280 goto err_create; 281 } 282 283 meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total), 284 GFP_KERNEL); 285 if (!meters_obj) { 286 err = -ENOMEM; 287 goto err_mem; 288 } 289 290 meters_obj->base_id = id; 291 meters_obj->total_meters = total; 292 list_add(&meters_obj->entry, &flow_meters->partial_list); 293 pos = 0; 294 } else { 295 pos = find_first_zero_bit(meters_obj->meters_map, total); 296 if (bitmap_weight(meters_obj->meters_map, total) == total - 1) { 297 list_del(&meters_obj->entry); 298 list_add(&meters_obj->entry, &flow_meters->full_list); 299 } 300 } 301 302 bitmap_set(meters_obj->meters_map, pos, 1); 303 meter->flow_meters = flow_meters; 304 meter->meters_obj = meters_obj; 305 meter->obj_id = meters_obj->base_id + pos / 2; 306 meter->idx = pos % 2; 307 308 mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n", 309 meter->obj_id, meter->idx); 310 311 return meter; 312 313 err_mem: 314 mlx5e_flow_meter_destroy_aso_obj(mdev, id); 315 err_create: 316 mlx5_fc_destroy(mdev, meter->green_counter); 317 err_green_counter: 318 mlx5_fc_destroy(mdev, meter->red_counter); 319 err_red_counter: 320 kfree(meter); 321 return ERR_PTR(err); 322 } 323 324 static void 325 __mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) 326 { 327 struct mlx5e_flow_meters *flow_meters = meter->flow_meters; 328 struct mlx5_core_dev *mdev = flow_meters->mdev; 329 struct mlx5e_flow_meter_aso_obj *meters_obj; 330 int n, pos; 331 332 mlx5_fc_destroy(mdev, meter->green_counter); 333 mlx5_fc_destroy(mdev, meter->red_counter); 334 335 meters_obj = meter->meters_obj; 336 pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx; 337 bitmap_clear(meters_obj->meters_map, pos, 1); 338 n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters); 339 if (n == 0) { 340 list_del(&meters_obj->entry); 341 mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id); 342 kfree(meters_obj); 343 } else if (n == meters_obj->total_meters - 1) { 344 list_del(&meters_obj->entry); 345 list_add(&meters_obj->entry, &flow_meters->partial_list); 346 } 347 348 mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n", 349 meter->obj_id, meter->idx); 350 kfree(meter); 351 } 352 353 static struct mlx5e_flow_meter_handle * 354 __mlx5e_tc_meter_get(struct mlx5e_flow_meters *flow_meters, u32 index) 355 { 356 struct mlx5e_flow_meter_handle *meter; 357 358 hash_for_each_possible(flow_meters->hashtbl, meter, hlist, index) 359 if (meter->params.index == index) 360 goto add_ref; 361 362 return ERR_PTR(-ENOENT); 363 364 add_ref: 365 meter->refcnt++; 366 367 return meter; 368 } 369 370 struct mlx5e_flow_meter_handle * 371 mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) 372 { 373 struct mlx5e_flow_meters *flow_meters; 374 struct mlx5e_flow_meter_handle *meter; 375 376 flow_meters = mlx5e_get_flow_meters(mdev); 377 if (!flow_meters) 378 return ERR_PTR(-EOPNOTSUPP); 379 380 mutex_lock(&flow_meters->sync_lock); 381 meter = __mlx5e_tc_meter_get(flow_meters, params->index); 382 mutex_unlock(&flow_meters->sync_lock); 383 384 return meter; 385 } 386 387 static void 388 __mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) 389 { 390 if (--meter->refcnt == 0) { 391 hash_del(&meter->hlist); 392 __mlx5e_flow_meter_free(meter); 393 } 394 } 395 396 void 397 mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) 398 { 399 struct mlx5e_flow_meters *flow_meters = meter->flow_meters; 400 401 mutex_lock(&flow_meters->sync_lock); 402 __mlx5e_tc_meter_put(meter); 403 mutex_unlock(&flow_meters->sync_lock); 404 } 405 406 static struct mlx5e_flow_meter_handle * 407 mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters, 408 struct mlx5e_flow_meter_params *params) 409 { 410 struct mlx5e_flow_meter_handle *meter; 411 412 meter = __mlx5e_flow_meter_alloc(flow_meters); 413 if (IS_ERR(meter)) 414 return meter; 415 416 hash_add(flow_meters->hashtbl, &meter->hlist, params->index); 417 meter->params.index = params->index; 418 meter->refcnt++; 419 420 return meter; 421 } 422 423 static int 424 __mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, 425 struct mlx5e_flow_meter_params *params) 426 { 427 struct mlx5_core_dev *mdev = meter->flow_meters->mdev; 428 int err = 0; 429 430 if (meter->params.mode != params->mode || meter->params.rate != params->rate || 431 meter->params.burst != params->burst) { 432 err = mlx5e_tc_meter_modify(mdev, meter, params); 433 if (err) 434 goto out; 435 436 meter->params.mode = params->mode; 437 meter->params.rate = params->rate; 438 meter->params.burst = params->burst; 439 } 440 441 out: 442 return err; 443 } 444 445 int 446 mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, 447 struct mlx5e_flow_meter_params *params) 448 { 449 struct mlx5_core_dev *mdev = meter->flow_meters->mdev; 450 struct mlx5e_flow_meters *flow_meters; 451 int err; 452 453 flow_meters = mlx5e_get_flow_meters(mdev); 454 if (!flow_meters) 455 return -EOPNOTSUPP; 456 457 mutex_lock(&flow_meters->sync_lock); 458 err = __mlx5e_tc_meter_update(meter, params); 459 mutex_unlock(&flow_meters->sync_lock); 460 return err; 461 } 462 463 struct mlx5e_flow_meter_handle * 464 mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) 465 { 466 struct mlx5e_flow_meters *flow_meters; 467 struct mlx5e_flow_meter_handle *meter; 468 int err; 469 470 flow_meters = mlx5e_get_flow_meters(mdev); 471 if (!flow_meters) 472 return ERR_PTR(-EOPNOTSUPP); 473 474 mutex_lock(&flow_meters->sync_lock); 475 meter = __mlx5e_tc_meter_get(flow_meters, params->index); 476 if (IS_ERR(meter)) { 477 meter = mlx5e_tc_meter_alloc(flow_meters, params); 478 if (IS_ERR(meter)) { 479 err = PTR_ERR(meter); 480 goto err_get; 481 } 482 } 483 484 err = __mlx5e_tc_meter_update(meter, params); 485 if (err) 486 goto err_update; 487 488 mutex_unlock(&flow_meters->sync_lock); 489 return meter; 490 491 err_update: 492 __mlx5e_tc_meter_put(meter); 493 err_get: 494 mutex_unlock(&flow_meters->sync_lock); 495 return ERR_PTR(err); 496 } 497 498 enum mlx5_flow_namespace_type 499 mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters) 500 { 501 return flow_meters->ns_type; 502 } 503 504 struct mlx5e_flow_meters * 505 mlx5e_flow_meters_init(struct mlx5e_priv *priv, 506 enum mlx5_flow_namespace_type ns_type, 507 struct mlx5e_post_act *post_act) 508 { 509 struct mlx5_core_dev *mdev = priv->mdev; 510 struct mlx5e_flow_meters *flow_meters; 511 int err; 512 513 if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & 514 MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO)) 515 return ERR_PTR(-EOPNOTSUPP); 516 517 if (IS_ERR_OR_NULL(post_act)) { 518 netdev_dbg(priv->netdev, 519 "flow meter offload is not supported, post action is missing\n"); 520 return ERR_PTR(-EOPNOTSUPP); 521 } 522 523 flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL); 524 if (!flow_meters) 525 return ERR_PTR(-ENOMEM); 526 527 err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn); 528 if (err) { 529 mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err); 530 goto err_out; 531 } 532 533 flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn); 534 if (IS_ERR(flow_meters->aso)) { 535 mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n"); 536 err = PTR_ERR(flow_meters->aso); 537 goto err_sq; 538 } 539 540 mutex_init(&flow_meters->sync_lock); 541 INIT_LIST_HEAD(&flow_meters->partial_list); 542 INIT_LIST_HEAD(&flow_meters->full_list); 543 544 flow_meters->ns_type = ns_type; 545 flow_meters->mdev = mdev; 546 flow_meters->post_act = post_act; 547 mutex_init(&flow_meters->aso_lock); 548 flow_meters->log_granularity = min_t(int, 6, 549 MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc)); 550 551 return flow_meters; 552 553 err_sq: 554 mlx5_core_dealloc_pd(mdev, flow_meters->pdn); 555 err_out: 556 kfree(flow_meters); 557 return ERR_PTR(err); 558 } 559 560 void 561 mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters) 562 { 563 if (IS_ERR_OR_NULL(flow_meters)) 564 return; 565 566 mlx5_aso_destroy(flow_meters->aso); 567 mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn); 568 kfree(flow_meters); 569 } 570 571 void 572 mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter, 573 u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse) 574 { 575 u64 bytes1, packets1, lastuse1; 576 u64 bytes2, packets2, lastuse2; 577 578 mlx5_fc_query_cached(meter->green_counter, &bytes1, &packets1, &lastuse1); 579 mlx5_fc_query_cached(meter->red_counter, &bytes2, &packets2, &lastuse2); 580 581 *bytes = bytes1 + bytes2; 582 *packets = packets1 + packets2; 583 *drops = packets2; 584 *lastuse = max_t(u64, lastuse1, lastuse2); 585 } 586