1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "dr_types.h" 5 6 #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64 7 8 struct mlx5dr_icm_pool { 9 enum mlx5dr_icm_type icm_type; 10 enum mlx5dr_icm_chunk_size max_log_chunk_sz; 11 struct mlx5dr_domain *dmn; 12 /* memory management */ 13 struct mutex mutex; /* protect the ICM pool and ICM buddy */ 14 struct list_head buddy_mem_list; 15 u64 hot_memory_size; 16 }; 17 18 struct mlx5dr_icm_dm { 19 u32 obj_id; 20 enum mlx5_sw_icm_type type; 21 phys_addr_t addr; 22 size_t length; 23 }; 24 25 struct mlx5dr_icm_mr { 26 u32 mkey; 27 struct mlx5dr_icm_dm dm; 28 struct mlx5dr_domain *dmn; 29 size_t length; 30 u64 icm_start_addr; 31 }; 32 33 static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev, 34 u32 pd, u64 length, u64 start_addr, int mode, 35 u32 *mkey) 36 { 37 u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 38 u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; 39 void *mkc; 40 41 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 42 43 MLX5_SET(mkc, mkc, access_mode_1_0, mode); 44 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 45 MLX5_SET(mkc, mkc, lw, 1); 46 MLX5_SET(mkc, mkc, lr, 1); 47 if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) { 48 MLX5_SET(mkc, mkc, rw, 1); 49 MLX5_SET(mkc, mkc, rr, 1); 50 } 51 52 MLX5_SET64(mkc, mkc, len, length); 53 MLX5_SET(mkc, mkc, pd, pd); 54 MLX5_SET(mkc, mkc, qpn, 0xffffff); 55 MLX5_SET64(mkc, mkc, start_addr, start_addr); 56 57 return mlx5_core_create_mkey(mdev, mkey, in, inlen); 58 } 59 60 static struct mlx5dr_icm_mr * 61 dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool) 62 { 63 struct mlx5_core_dev *mdev = pool->dmn->mdev; 64 enum mlx5_sw_icm_type dm_type; 65 struct mlx5dr_icm_mr *icm_mr; 66 size_t log_align_base; 67 int err; 68 69 icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL); 70 if (!icm_mr) 71 return NULL; 72 73 icm_mr->dmn = pool->dmn; 74 75 icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 76 pool->icm_type); 77 78 if (pool->icm_type == DR_ICM_TYPE_STE) { 79 dm_type = MLX5_SW_ICM_TYPE_STEERING; 80 log_align_base = ilog2(icm_mr->dm.length); 81 } else { 82 dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY; 83 /* Align base is 64B */ 84 log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE); 85 } 86 icm_mr->dm.type = dm_type; 87 88 err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 89 log_align_base, 0, &icm_mr->dm.addr, 90 &icm_mr->dm.obj_id); 91 if (err) { 92 mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err); 93 goto free_icm_mr; 94 } 95 96 /* Register device memory */ 97 err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn, 98 icm_mr->dm.length, 99 icm_mr->dm.addr, 100 MLX5_MKC_ACCESS_MODE_SW_ICM, 101 &icm_mr->mkey); 102 if (err) { 103 mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err); 104 goto free_dm; 105 } 106 107 icm_mr->icm_start_addr = icm_mr->dm.addr; 108 109 if (icm_mr->icm_start_addr & (BIT(log_align_base) - 1)) { 110 mlx5dr_err(pool->dmn, "Failed to get Aligned ICM mem (asked: %zu)\n", 111 log_align_base); 112 goto free_mkey; 113 } 114 115 return icm_mr; 116 117 free_mkey: 118 mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 119 free_dm: 120 mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0, 121 icm_mr->dm.addr, icm_mr->dm.obj_id); 122 free_icm_mr: 123 kvfree(icm_mr); 124 return NULL; 125 } 126 127 static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr) 128 { 129 struct mlx5_core_dev *mdev = icm_mr->dmn->mdev; 130 struct mlx5dr_icm_dm *dm = &icm_mr->dm; 131 132 mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 133 mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0, 134 dm->addr, dm->obj_id); 135 kvfree(icm_mr); 136 } 137 138 static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy) 139 { 140 /* We support only one type of STE size, both for ConnectX-5 and later 141 * devices. Once the support for match STE which has a larger tag is 142 * added (32B instead of 16B), the STE size for devices later than 143 * ConnectX-5 needs to account for that. 144 */ 145 return DR_STE_SIZE_REDUCED; 146 } 147 148 static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset) 149 { 150 struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 151 int index = offset / DR_STE_SIZE; 152 153 chunk->ste_arr = &buddy->ste_arr[index]; 154 chunk->miss_list = &buddy->miss_list[index]; 155 chunk->hw_ste_arr = buddy->hw_ste_arr + 156 index * dr_icm_buddy_get_ste_size(buddy); 157 } 158 159 static void dr_icm_chunk_ste_cleanup(struct mlx5dr_icm_chunk *chunk) 160 { 161 struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 162 163 memset(chunk->hw_ste_arr, 0, 164 chunk->num_of_entries * dr_icm_buddy_get_ste_size(buddy)); 165 memset(chunk->ste_arr, 0, 166 chunk->num_of_entries * sizeof(chunk->ste_arr[0])); 167 } 168 169 static enum mlx5dr_icm_type 170 get_chunk_icm_type(struct mlx5dr_icm_chunk *chunk) 171 { 172 return chunk->buddy_mem->pool->icm_type; 173 } 174 175 static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk, 176 struct mlx5dr_icm_buddy_mem *buddy) 177 { 178 enum mlx5dr_icm_type icm_type = get_chunk_icm_type(chunk); 179 180 buddy->used_memory -= chunk->byte_size; 181 list_del(&chunk->chunk_list); 182 183 if (icm_type == DR_ICM_TYPE_STE) 184 dr_icm_chunk_ste_cleanup(chunk); 185 186 kvfree(chunk); 187 } 188 189 static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 190 { 191 int num_of_entries = 192 mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz); 193 194 buddy->ste_arr = kvcalloc(num_of_entries, 195 sizeof(struct mlx5dr_ste), GFP_KERNEL); 196 if (!buddy->ste_arr) 197 return -ENOMEM; 198 199 /* Preallocate full STE size on non-ConnectX-5 devices since 200 * we need to support both full and reduced with the same cache. 201 */ 202 buddy->hw_ste_arr = kvcalloc(num_of_entries, 203 dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL); 204 if (!buddy->hw_ste_arr) 205 goto free_ste_arr; 206 207 buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL); 208 if (!buddy->miss_list) 209 goto free_hw_ste_arr; 210 211 return 0; 212 213 free_hw_ste_arr: 214 kvfree(buddy->hw_ste_arr); 215 free_ste_arr: 216 kvfree(buddy->ste_arr); 217 return -ENOMEM; 218 } 219 220 static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 221 { 222 kvfree(buddy->ste_arr); 223 kvfree(buddy->hw_ste_arr); 224 kvfree(buddy->miss_list); 225 } 226 227 static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool) 228 { 229 struct mlx5dr_icm_buddy_mem *buddy; 230 struct mlx5dr_icm_mr *icm_mr; 231 232 icm_mr = dr_icm_pool_mr_create(pool); 233 if (!icm_mr) 234 return -ENOMEM; 235 236 buddy = kvzalloc(sizeof(*buddy), GFP_KERNEL); 237 if (!buddy) 238 goto free_mr; 239 240 if (mlx5dr_buddy_init(buddy, pool->max_log_chunk_sz)) 241 goto err_free_buddy; 242 243 buddy->icm_mr = icm_mr; 244 buddy->pool = pool; 245 246 if (pool->icm_type == DR_ICM_TYPE_STE) { 247 /* Reduce allocations by preallocating and reusing the STE structures */ 248 if (dr_icm_buddy_init_ste_cache(buddy)) 249 goto err_cleanup_buddy; 250 } 251 252 /* add it to the -start- of the list in order to search in it first */ 253 list_add(&buddy->list_node, &pool->buddy_mem_list); 254 255 return 0; 256 257 err_cleanup_buddy: 258 mlx5dr_buddy_cleanup(buddy); 259 err_free_buddy: 260 kvfree(buddy); 261 free_mr: 262 dr_icm_pool_mr_destroy(icm_mr); 263 return -ENOMEM; 264 } 265 266 static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy) 267 { 268 struct mlx5dr_icm_chunk *chunk, *next; 269 270 list_for_each_entry_safe(chunk, next, &buddy->hot_list, chunk_list) 271 dr_icm_chunk_destroy(chunk, buddy); 272 273 list_for_each_entry_safe(chunk, next, &buddy->used_list, chunk_list) 274 dr_icm_chunk_destroy(chunk, buddy); 275 276 dr_icm_pool_mr_destroy(buddy->icm_mr); 277 278 mlx5dr_buddy_cleanup(buddy); 279 280 if (buddy->pool->icm_type == DR_ICM_TYPE_STE) 281 dr_icm_buddy_cleanup_ste_cache(buddy); 282 283 kvfree(buddy); 284 } 285 286 static struct mlx5dr_icm_chunk * 287 dr_icm_chunk_create(struct mlx5dr_icm_pool *pool, 288 enum mlx5dr_icm_chunk_size chunk_size, 289 struct mlx5dr_icm_buddy_mem *buddy_mem_pool, 290 unsigned int seg) 291 { 292 struct mlx5dr_icm_chunk *chunk; 293 int offset; 294 295 chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL); 296 if (!chunk) 297 return NULL; 298 299 offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg; 300 301 chunk->rkey = buddy_mem_pool->icm_mr->mkey; 302 chunk->mr_addr = offset; 303 chunk->icm_addr = 304 (uintptr_t)buddy_mem_pool->icm_mr->icm_start_addr + offset; 305 chunk->num_of_entries = 306 mlx5dr_icm_pool_chunk_size_to_entries(chunk_size); 307 chunk->byte_size = 308 mlx5dr_icm_pool_chunk_size_to_byte(chunk_size, pool->icm_type); 309 chunk->seg = seg; 310 chunk->buddy_mem = buddy_mem_pool; 311 312 if (pool->icm_type == DR_ICM_TYPE_STE) 313 dr_icm_chunk_ste_init(chunk, offset); 314 315 buddy_mem_pool->used_memory += chunk->byte_size; 316 INIT_LIST_HEAD(&chunk->chunk_list); 317 318 /* chunk now is part of the used_list */ 319 list_add_tail(&chunk->chunk_list, &buddy_mem_pool->used_list); 320 321 return chunk; 322 } 323 324 static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool) 325 { 326 int allow_hot_size; 327 328 /* sync when hot memory reaches half of the pool size */ 329 allow_hot_size = 330 mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 331 pool->icm_type) / 2; 332 333 return pool->hot_memory_size > allow_hot_size; 334 } 335 336 static int dr_icm_pool_sync_all_buddy_pools(struct mlx5dr_icm_pool *pool) 337 { 338 struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 339 int err; 340 341 err = mlx5dr_cmd_sync_steering(pool->dmn->mdev); 342 if (err) { 343 mlx5dr_err(pool->dmn, "Failed to sync to HW (err: %d)\n", err); 344 return err; 345 } 346 347 list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) { 348 struct mlx5dr_icm_chunk *chunk, *tmp_chunk; 349 350 list_for_each_entry_safe(chunk, tmp_chunk, &buddy->hot_list, chunk_list) { 351 mlx5dr_buddy_free_mem(buddy, chunk->seg, 352 ilog2(chunk->num_of_entries)); 353 pool->hot_memory_size -= chunk->byte_size; 354 dr_icm_chunk_destroy(chunk, buddy); 355 } 356 357 if (!buddy->used_memory && pool->icm_type == DR_ICM_TYPE_STE) 358 dr_icm_buddy_destroy(buddy); 359 } 360 361 return 0; 362 } 363 364 static int dr_icm_handle_buddies_get_mem(struct mlx5dr_icm_pool *pool, 365 enum mlx5dr_icm_chunk_size chunk_size, 366 struct mlx5dr_icm_buddy_mem **buddy, 367 unsigned int *seg) 368 { 369 struct mlx5dr_icm_buddy_mem *buddy_mem_pool; 370 bool new_mem = false; 371 int err; 372 373 alloc_buddy_mem: 374 /* find the next free place from the buddy list */ 375 list_for_each_entry(buddy_mem_pool, &pool->buddy_mem_list, list_node) { 376 err = mlx5dr_buddy_alloc_mem(buddy_mem_pool, 377 chunk_size, seg); 378 if (!err) 379 goto found; 380 381 if (WARN_ON(new_mem)) { 382 /* We have new memory pool, first in the list */ 383 mlx5dr_err(pool->dmn, 384 "No memory for order: %d\n", 385 chunk_size); 386 goto out; 387 } 388 } 389 390 /* no more available allocators in that pool, create new */ 391 err = dr_icm_buddy_create(pool); 392 if (err) { 393 mlx5dr_err(pool->dmn, 394 "Failed creating buddy for order %d\n", 395 chunk_size); 396 goto out; 397 } 398 399 /* mark we have new memory, first in list */ 400 new_mem = true; 401 goto alloc_buddy_mem; 402 403 found: 404 *buddy = buddy_mem_pool; 405 out: 406 return err; 407 } 408 409 /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and 410 * also memory used for HW STE management for optimizations. 411 */ 412 struct mlx5dr_icm_chunk * 413 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 414 enum mlx5dr_icm_chunk_size chunk_size) 415 { 416 struct mlx5dr_icm_chunk *chunk = NULL; 417 struct mlx5dr_icm_buddy_mem *buddy; 418 unsigned int seg; 419 int ret; 420 421 if (chunk_size > pool->max_log_chunk_sz) 422 return NULL; 423 424 mutex_lock(&pool->mutex); 425 /* find mem, get back the relevant buddy pool and seg in that mem */ 426 ret = dr_icm_handle_buddies_get_mem(pool, chunk_size, &buddy, &seg); 427 if (ret) 428 goto out; 429 430 chunk = dr_icm_chunk_create(pool, chunk_size, buddy, seg); 431 if (!chunk) 432 goto out_err; 433 434 goto out; 435 436 out_err: 437 mlx5dr_buddy_free_mem(buddy, seg, chunk_size); 438 out: 439 mutex_unlock(&pool->mutex); 440 return chunk; 441 } 442 443 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk) 444 { 445 struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 446 struct mlx5dr_icm_pool *pool = buddy->pool; 447 448 /* move the memory to the waiting list AKA "hot" */ 449 mutex_lock(&pool->mutex); 450 list_move_tail(&chunk->chunk_list, &buddy->hot_list); 451 pool->hot_memory_size += chunk->byte_size; 452 453 /* Check if we have chunks that are waiting for sync-ste */ 454 if (dr_icm_pool_is_sync_required(pool)) 455 dr_icm_pool_sync_all_buddy_pools(pool); 456 457 mutex_unlock(&pool->mutex); 458 } 459 460 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 461 enum mlx5dr_icm_type icm_type) 462 { 463 enum mlx5dr_icm_chunk_size max_log_chunk_sz; 464 struct mlx5dr_icm_pool *pool; 465 466 if (icm_type == DR_ICM_TYPE_STE) 467 max_log_chunk_sz = dmn->info.max_log_sw_icm_sz; 468 else 469 max_log_chunk_sz = dmn->info.max_log_action_icm_sz; 470 471 pool = kvzalloc(sizeof(*pool), GFP_KERNEL); 472 if (!pool) 473 return NULL; 474 475 pool->dmn = dmn; 476 pool->icm_type = icm_type; 477 pool->max_log_chunk_sz = max_log_chunk_sz; 478 479 INIT_LIST_HEAD(&pool->buddy_mem_list); 480 481 mutex_init(&pool->mutex); 482 483 return pool; 484 } 485 486 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool) 487 { 488 struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 489 490 list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) 491 dr_icm_buddy_destroy(buddy); 492 493 mutex_destroy(&pool->mutex); 494 kvfree(pool); 495 } 496