1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "dr_types.h" 5 6 #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64 7 #define DR_ICM_POOL_HOT_MEMORY_FRACTION 4 8 9 struct mlx5dr_icm_hot_chunk { 10 struct mlx5dr_icm_buddy_mem *buddy_mem; 11 unsigned int seg; 12 enum mlx5dr_icm_chunk_size size; 13 }; 14 15 struct mlx5dr_icm_pool { 16 enum mlx5dr_icm_type icm_type; 17 enum mlx5dr_icm_chunk_size max_log_chunk_sz; 18 struct mlx5dr_domain *dmn; 19 struct kmem_cache *chunks_kmem_cache; 20 21 /* memory management */ 22 struct mutex mutex; /* protect the ICM pool and ICM buddy */ 23 struct list_head buddy_mem_list; 24 25 /* Hardware may be accessing this memory but at some future, 26 * undetermined time, it might cease to do so. 27 * sync_ste command sets them free. 28 */ 29 struct mlx5dr_icm_hot_chunk *hot_chunks_arr; 30 u32 hot_chunks_num; 31 u64 hot_memory_size; 32 }; 33 34 struct mlx5dr_icm_dm { 35 u32 obj_id; 36 enum mlx5_sw_icm_type type; 37 phys_addr_t addr; 38 size_t length; 39 }; 40 41 struct mlx5dr_icm_mr { 42 u32 mkey; 43 struct mlx5dr_icm_dm dm; 44 struct mlx5dr_domain *dmn; 45 size_t length; 46 u64 icm_start_addr; 47 }; 48 49 static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev, 50 u32 pd, u64 length, u64 start_addr, int mode, 51 u32 *mkey) 52 { 53 u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 54 u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; 55 void *mkc; 56 57 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 58 59 MLX5_SET(mkc, mkc, access_mode_1_0, mode); 60 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 61 MLX5_SET(mkc, mkc, lw, 1); 62 MLX5_SET(mkc, mkc, lr, 1); 63 if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) { 64 MLX5_SET(mkc, mkc, rw, 1); 65 MLX5_SET(mkc, mkc, rr, 1); 66 } 67 68 MLX5_SET64(mkc, mkc, len, length); 69 MLX5_SET(mkc, mkc, pd, pd); 70 MLX5_SET(mkc, mkc, qpn, 0xffffff); 71 MLX5_SET64(mkc, mkc, start_addr, start_addr); 72 73 return mlx5_core_create_mkey(mdev, mkey, in, inlen); 74 } 75 76 u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk) 77 { 78 u32 offset = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); 79 80 return (u64)offset * chunk->seg; 81 } 82 83 u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk) 84 { 85 return chunk->buddy_mem->icm_mr->mkey; 86 } 87 88 u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk) 89 { 90 u32 size = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); 91 92 return (u64)chunk->buddy_mem->icm_mr->icm_start_addr + size * chunk->seg; 93 } 94 95 u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk) 96 { 97 return mlx5dr_icm_pool_chunk_size_to_byte(chunk->size, 98 chunk->buddy_mem->pool->icm_type); 99 } 100 101 u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk) 102 { 103 return mlx5dr_icm_pool_chunk_size_to_entries(chunk->size); 104 } 105 106 static struct mlx5dr_icm_mr * 107 dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool) 108 { 109 struct mlx5_core_dev *mdev = pool->dmn->mdev; 110 enum mlx5_sw_icm_type dm_type; 111 struct mlx5dr_icm_mr *icm_mr; 112 size_t log_align_base; 113 int err; 114 115 icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL); 116 if (!icm_mr) 117 return NULL; 118 119 icm_mr->dmn = pool->dmn; 120 121 icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 122 pool->icm_type); 123 124 if (pool->icm_type == DR_ICM_TYPE_STE) { 125 dm_type = MLX5_SW_ICM_TYPE_STEERING; 126 log_align_base = ilog2(icm_mr->dm.length); 127 } else { 128 dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY; 129 /* Align base is 64B */ 130 log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE); 131 } 132 icm_mr->dm.type = dm_type; 133 134 err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 135 log_align_base, 0, &icm_mr->dm.addr, 136 &icm_mr->dm.obj_id); 137 if (err) { 138 mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err); 139 goto free_icm_mr; 140 } 141 142 /* Register device memory */ 143 err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn, 144 icm_mr->dm.length, 145 icm_mr->dm.addr, 146 MLX5_MKC_ACCESS_MODE_SW_ICM, 147 &icm_mr->mkey); 148 if (err) { 149 mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err); 150 goto free_dm; 151 } 152 153 icm_mr->icm_start_addr = icm_mr->dm.addr; 154 155 if (icm_mr->icm_start_addr & (BIT(log_align_base) - 1)) { 156 mlx5dr_err(pool->dmn, "Failed to get Aligned ICM mem (asked: %zu)\n", 157 log_align_base); 158 goto free_mkey; 159 } 160 161 return icm_mr; 162 163 free_mkey: 164 mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 165 free_dm: 166 mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0, 167 icm_mr->dm.addr, icm_mr->dm.obj_id); 168 free_icm_mr: 169 kvfree(icm_mr); 170 return NULL; 171 } 172 173 static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr) 174 { 175 struct mlx5_core_dev *mdev = icm_mr->dmn->mdev; 176 struct mlx5dr_icm_dm *dm = &icm_mr->dm; 177 178 mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 179 mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0, 180 dm->addr, dm->obj_id); 181 kvfree(icm_mr); 182 } 183 184 static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy) 185 { 186 /* We support only one type of STE size, both for ConnectX-5 and later 187 * devices. Once the support for match STE which has a larger tag is 188 * added (32B instead of 16B), the STE size for devices later than 189 * ConnectX-5 needs to account for that. 190 */ 191 return DR_STE_SIZE_REDUCED; 192 } 193 194 static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset) 195 { 196 int num_of_entries = mlx5dr_icm_pool_get_chunk_num_of_entries(chunk); 197 struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 198 int ste_size = dr_icm_buddy_get_ste_size(buddy); 199 int index = offset / DR_STE_SIZE; 200 201 chunk->ste_arr = &buddy->ste_arr[index]; 202 chunk->miss_list = &buddy->miss_list[index]; 203 chunk->hw_ste_arr = buddy->hw_ste_arr + index * ste_size; 204 205 memset(chunk->hw_ste_arr, 0, num_of_entries * ste_size); 206 memset(chunk->ste_arr, 0, 207 num_of_entries * sizeof(chunk->ste_arr[0])); 208 } 209 210 static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 211 { 212 int num_of_entries = 213 mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz); 214 215 buddy->ste_arr = kvcalloc(num_of_entries, 216 sizeof(struct mlx5dr_ste), GFP_KERNEL); 217 if (!buddy->ste_arr) 218 return -ENOMEM; 219 220 /* Preallocate full STE size on non-ConnectX-5 devices since 221 * we need to support both full and reduced with the same cache. 222 */ 223 buddy->hw_ste_arr = kvcalloc(num_of_entries, 224 dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL); 225 if (!buddy->hw_ste_arr) 226 goto free_ste_arr; 227 228 buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL); 229 if (!buddy->miss_list) 230 goto free_hw_ste_arr; 231 232 return 0; 233 234 free_hw_ste_arr: 235 kvfree(buddy->hw_ste_arr); 236 free_ste_arr: 237 kvfree(buddy->ste_arr); 238 return -ENOMEM; 239 } 240 241 static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 242 { 243 kvfree(buddy->ste_arr); 244 kvfree(buddy->hw_ste_arr); 245 kvfree(buddy->miss_list); 246 } 247 248 static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool) 249 { 250 struct mlx5dr_icm_buddy_mem *buddy; 251 struct mlx5dr_icm_mr *icm_mr; 252 253 icm_mr = dr_icm_pool_mr_create(pool); 254 if (!icm_mr) 255 return -ENOMEM; 256 257 buddy = kvzalloc(sizeof(*buddy), GFP_KERNEL); 258 if (!buddy) 259 goto free_mr; 260 261 if (mlx5dr_buddy_init(buddy, pool->max_log_chunk_sz)) 262 goto err_free_buddy; 263 264 buddy->icm_mr = icm_mr; 265 buddy->pool = pool; 266 267 if (pool->icm_type == DR_ICM_TYPE_STE) { 268 /* Reduce allocations by preallocating and reusing the STE structures */ 269 if (dr_icm_buddy_init_ste_cache(buddy)) 270 goto err_cleanup_buddy; 271 } 272 273 /* add it to the -start- of the list in order to search in it first */ 274 list_add(&buddy->list_node, &pool->buddy_mem_list); 275 276 return 0; 277 278 err_cleanup_buddy: 279 mlx5dr_buddy_cleanup(buddy); 280 err_free_buddy: 281 kvfree(buddy); 282 free_mr: 283 dr_icm_pool_mr_destroy(icm_mr); 284 return -ENOMEM; 285 } 286 287 static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy) 288 { 289 dr_icm_pool_mr_destroy(buddy->icm_mr); 290 291 mlx5dr_buddy_cleanup(buddy); 292 293 if (buddy->pool->icm_type == DR_ICM_TYPE_STE) 294 dr_icm_buddy_cleanup_ste_cache(buddy); 295 296 kvfree(buddy); 297 } 298 299 static void 300 dr_icm_chunk_init(struct mlx5dr_icm_chunk *chunk, 301 struct mlx5dr_icm_pool *pool, 302 enum mlx5dr_icm_chunk_size chunk_size, 303 struct mlx5dr_icm_buddy_mem *buddy_mem_pool, 304 unsigned int seg) 305 { 306 int offset; 307 308 chunk->seg = seg; 309 chunk->size = chunk_size; 310 chunk->buddy_mem = buddy_mem_pool; 311 312 if (pool->icm_type == DR_ICM_TYPE_STE) { 313 offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg; 314 dr_icm_chunk_ste_init(chunk, offset); 315 } 316 317 buddy_mem_pool->used_memory += mlx5dr_icm_pool_get_chunk_byte_size(chunk); 318 } 319 320 static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool) 321 { 322 int allow_hot_size; 323 324 /* sync when hot memory reaches a certain fraction of the pool size */ 325 allow_hot_size = 326 mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 327 pool->icm_type) / 328 DR_ICM_POOL_HOT_MEMORY_FRACTION; 329 330 return pool->hot_memory_size > allow_hot_size; 331 } 332 333 static void dr_icm_pool_clear_hot_chunks_arr(struct mlx5dr_icm_pool *pool) 334 { 335 struct mlx5dr_icm_hot_chunk *hot_chunk; 336 u32 i, num_entries; 337 338 for (i = 0; i < pool->hot_chunks_num; i++) { 339 hot_chunk = &pool->hot_chunks_arr[i]; 340 num_entries = mlx5dr_icm_pool_chunk_size_to_entries(hot_chunk->size); 341 mlx5dr_buddy_free_mem(hot_chunk->buddy_mem, 342 hot_chunk->seg, ilog2(num_entries)); 343 hot_chunk->buddy_mem->used_memory -= 344 mlx5dr_icm_pool_chunk_size_to_byte(hot_chunk->size, 345 pool->icm_type); 346 } 347 348 pool->hot_chunks_num = 0; 349 pool->hot_memory_size = 0; 350 } 351 352 static int dr_icm_pool_sync_all_buddy_pools(struct mlx5dr_icm_pool *pool) 353 { 354 struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 355 int err; 356 357 err = mlx5dr_cmd_sync_steering(pool->dmn->mdev); 358 if (err) { 359 mlx5dr_err(pool->dmn, "Failed to sync to HW (err: %d)\n", err); 360 return err; 361 } 362 363 dr_icm_pool_clear_hot_chunks_arr(pool); 364 365 list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) { 366 if (!buddy->used_memory && pool->icm_type == DR_ICM_TYPE_STE) 367 dr_icm_buddy_destroy(buddy); 368 } 369 370 return 0; 371 } 372 373 static int dr_icm_handle_buddies_get_mem(struct mlx5dr_icm_pool *pool, 374 enum mlx5dr_icm_chunk_size chunk_size, 375 struct mlx5dr_icm_buddy_mem **buddy, 376 unsigned int *seg) 377 { 378 struct mlx5dr_icm_buddy_mem *buddy_mem_pool; 379 bool new_mem = false; 380 int err; 381 382 alloc_buddy_mem: 383 /* find the next free place from the buddy list */ 384 list_for_each_entry(buddy_mem_pool, &pool->buddy_mem_list, list_node) { 385 err = mlx5dr_buddy_alloc_mem(buddy_mem_pool, 386 chunk_size, seg); 387 if (!err) 388 goto found; 389 390 if (WARN_ON(new_mem)) { 391 /* We have new memory pool, first in the list */ 392 mlx5dr_err(pool->dmn, 393 "No memory for order: %d\n", 394 chunk_size); 395 goto out; 396 } 397 } 398 399 /* no more available allocators in that pool, create new */ 400 err = dr_icm_buddy_create(pool); 401 if (err) { 402 mlx5dr_err(pool->dmn, 403 "Failed creating buddy for order %d\n", 404 chunk_size); 405 goto out; 406 } 407 408 /* mark we have new memory, first in list */ 409 new_mem = true; 410 goto alloc_buddy_mem; 411 412 found: 413 *buddy = buddy_mem_pool; 414 out: 415 return err; 416 } 417 418 /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and 419 * also memory used for HW STE management for optimizations. 420 */ 421 struct mlx5dr_icm_chunk * 422 mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 423 enum mlx5dr_icm_chunk_size chunk_size) 424 { 425 struct mlx5dr_icm_chunk *chunk = NULL; 426 struct mlx5dr_icm_buddy_mem *buddy; 427 unsigned int seg; 428 int ret; 429 430 if (chunk_size > pool->max_log_chunk_sz) 431 return NULL; 432 433 mutex_lock(&pool->mutex); 434 /* find mem, get back the relevant buddy pool and seg in that mem */ 435 ret = dr_icm_handle_buddies_get_mem(pool, chunk_size, &buddy, &seg); 436 if (ret) 437 goto out; 438 439 chunk = kmem_cache_alloc(pool->chunks_kmem_cache, GFP_KERNEL); 440 if (!chunk) 441 goto out_err; 442 443 dr_icm_chunk_init(chunk, pool, chunk_size, buddy, seg); 444 445 goto out; 446 447 out_err: 448 mlx5dr_buddy_free_mem(buddy, seg, chunk_size); 449 out: 450 mutex_unlock(&pool->mutex); 451 return chunk; 452 } 453 454 void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk) 455 { 456 struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 457 struct mlx5dr_icm_pool *pool = buddy->pool; 458 struct mlx5dr_icm_hot_chunk *hot_chunk; 459 struct kmem_cache *chunks_cache; 460 461 chunks_cache = pool->chunks_kmem_cache; 462 463 /* move the chunk to the waiting chunks array, AKA "hot" memory */ 464 mutex_lock(&pool->mutex); 465 466 pool->hot_memory_size += mlx5dr_icm_pool_get_chunk_byte_size(chunk); 467 468 hot_chunk = &pool->hot_chunks_arr[pool->hot_chunks_num++]; 469 hot_chunk->buddy_mem = chunk->buddy_mem; 470 hot_chunk->seg = chunk->seg; 471 hot_chunk->size = chunk->size; 472 473 kmem_cache_free(chunks_cache, chunk); 474 475 /* Check if we have chunks that are waiting for sync-ste */ 476 if (dr_icm_pool_is_sync_required(pool)) 477 dr_icm_pool_sync_all_buddy_pools(pool); 478 479 mutex_unlock(&pool->mutex); 480 } 481 482 struct mlx5dr_ste_htbl *mlx5dr_icm_pool_alloc_htbl(struct mlx5dr_icm_pool *pool) 483 { 484 return kmem_cache_alloc(pool->dmn->htbls_kmem_cache, GFP_KERNEL); 485 } 486 487 void mlx5dr_icm_pool_free_htbl(struct mlx5dr_icm_pool *pool, struct mlx5dr_ste_htbl *htbl) 488 { 489 kmem_cache_free(pool->dmn->htbls_kmem_cache, htbl); 490 } 491 492 struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 493 enum mlx5dr_icm_type icm_type) 494 { 495 u32 num_of_chunks, entry_size, max_hot_size; 496 enum mlx5dr_icm_chunk_size max_log_chunk_sz; 497 struct mlx5dr_icm_pool *pool; 498 499 if (icm_type == DR_ICM_TYPE_STE) 500 max_log_chunk_sz = dmn->info.max_log_sw_icm_sz; 501 else 502 max_log_chunk_sz = dmn->info.max_log_action_icm_sz; 503 504 pool = kvzalloc(sizeof(*pool), GFP_KERNEL); 505 if (!pool) 506 return NULL; 507 508 pool->dmn = dmn; 509 pool->icm_type = icm_type; 510 pool->max_log_chunk_sz = max_log_chunk_sz; 511 pool->chunks_kmem_cache = dmn->chunks_kmem_cache; 512 513 INIT_LIST_HEAD(&pool->buddy_mem_list); 514 515 mutex_init(&pool->mutex); 516 517 entry_size = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type); 518 519 max_hot_size = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 520 pool->icm_type) / 521 DR_ICM_POOL_HOT_MEMORY_FRACTION; 522 523 num_of_chunks = DIV_ROUND_UP(max_hot_size, entry_size) + 1; 524 525 pool->hot_chunks_arr = kvcalloc(num_of_chunks, 526 sizeof(struct mlx5dr_icm_hot_chunk), 527 GFP_KERNEL); 528 if (!pool->hot_chunks_arr) 529 goto free_pool; 530 531 return pool; 532 533 free_pool: 534 kvfree(pool); 535 return NULL; 536 } 537 538 void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool) 539 { 540 struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 541 542 dr_icm_pool_clear_hot_chunks_arr(pool); 543 544 list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) 545 dr_icm_buddy_destroy(buddy); 546 547 kvfree(pool->hot_chunks_arr); 548 mutex_destroy(&pool->mutex); 549 kvfree(pool); 550 } 551