129cf8febSAlex Vesker // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 229cf8febSAlex Vesker /* Copyright (c) 2019 Mellanox Technologies. */ 329cf8febSAlex Vesker 429cf8febSAlex Vesker #include "dr_types.h" 529cf8febSAlex Vesker 629cf8febSAlex Vesker #define DR_ICM_MODIFY_HDR_ALIGN_BASE 64 729cf8febSAlex Vesker 829cf8febSAlex Vesker struct mlx5dr_icm_pool { 929cf8febSAlex Vesker enum mlx5dr_icm_type icm_type; 1029cf8febSAlex Vesker enum mlx5dr_icm_chunk_size max_log_chunk_sz; 1129cf8febSAlex Vesker struct mlx5dr_domain *dmn; 12a00cd878SYevgeny Kliteynik /* memory management */ 13a00cd878SYevgeny Kliteynik struct mutex mutex; /* protect the ICM pool and ICM buddy */ 14a00cd878SYevgeny Kliteynik struct list_head buddy_mem_list; 151c586514SYevgeny Kliteynik u64 hot_memory_size; 1629cf8febSAlex Vesker }; 1729cf8febSAlex Vesker 1829cf8febSAlex Vesker struct mlx5dr_icm_dm { 1929cf8febSAlex Vesker u32 obj_id; 2029cf8febSAlex Vesker enum mlx5_sw_icm_type type; 21334a306fSNathan Chancellor phys_addr_t addr; 2229cf8febSAlex Vesker size_t length; 2329cf8febSAlex Vesker }; 2429cf8febSAlex Vesker 2529cf8febSAlex Vesker struct mlx5dr_icm_mr { 2683fec3f1SAharon Landau u32 mkey; 2729cf8febSAlex Vesker struct mlx5dr_icm_dm dm; 28a00cd878SYevgeny Kliteynik struct mlx5dr_domain *dmn; 2929cf8febSAlex Vesker size_t length; 3029cf8febSAlex Vesker u64 icm_start_addr; 3129cf8febSAlex Vesker }; 3229cf8febSAlex Vesker 3329cf8febSAlex Vesker static int dr_icm_create_dm_mkey(struct mlx5_core_dev *mdev, 3429cf8febSAlex Vesker u32 pd, u64 length, u64 start_addr, int mode, 3583fec3f1SAharon Landau u32 *mkey) 3629cf8febSAlex Vesker { 3729cf8febSAlex Vesker u32 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 3829cf8febSAlex Vesker u32 in[MLX5_ST_SZ_DW(create_mkey_in)] = {}; 3929cf8febSAlex Vesker void *mkc; 4029cf8febSAlex Vesker 4129cf8febSAlex Vesker mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 4229cf8febSAlex Vesker 4329cf8febSAlex Vesker MLX5_SET(mkc, mkc, access_mode_1_0, mode); 4429cf8febSAlex Vesker MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 4529cf8febSAlex Vesker MLX5_SET(mkc, mkc, lw, 1); 4629cf8febSAlex Vesker MLX5_SET(mkc, mkc, lr, 1); 4729cf8febSAlex Vesker if (mode == MLX5_MKC_ACCESS_MODE_SW_ICM) { 4829cf8febSAlex Vesker MLX5_SET(mkc, mkc, rw, 1); 4929cf8febSAlex Vesker MLX5_SET(mkc, mkc, rr, 1); 5029cf8febSAlex Vesker } 5129cf8febSAlex Vesker 5229cf8febSAlex Vesker MLX5_SET64(mkc, mkc, len, length); 5329cf8febSAlex Vesker MLX5_SET(mkc, mkc, pd, pd); 5429cf8febSAlex Vesker MLX5_SET(mkc, mkc, qpn, 0xffffff); 5529cf8febSAlex Vesker MLX5_SET64(mkc, mkc, start_addr, start_addr); 5629cf8febSAlex Vesker 5729cf8febSAlex Vesker return mlx5_core_create_mkey(mdev, mkey, in, inlen); 5829cf8febSAlex Vesker } 5929cf8febSAlex Vesker 60003f4f9aSRongwei Liu u64 mlx5dr_icm_pool_get_chunk_mr_addr(struct mlx5dr_icm_chunk *chunk) 61003f4f9aSRongwei Liu { 62003f4f9aSRongwei Liu u32 offset = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); 63003f4f9aSRongwei Liu 64003f4f9aSRongwei Liu return (u64)offset * chunk->seg; 65003f4f9aSRongwei Liu } 66003f4f9aSRongwei Liu 67003f4f9aSRongwei Liu u32 mlx5dr_icm_pool_get_chunk_rkey(struct mlx5dr_icm_chunk *chunk) 68003f4f9aSRongwei Liu { 69003f4f9aSRongwei Liu return chunk->buddy_mem->icm_mr->mkey; 70003f4f9aSRongwei Liu } 71003f4f9aSRongwei Liu 725c4f9b6eSRongwei Liu u64 mlx5dr_icm_pool_get_chunk_icm_addr(struct mlx5dr_icm_chunk *chunk) 735c4f9b6eSRongwei Liu { 745c4f9b6eSRongwei Liu u32 size = mlx5dr_icm_pool_dm_type_to_entry_size(chunk->buddy_mem->pool->icm_type); 755c4f9b6eSRongwei Liu 765c4f9b6eSRongwei Liu return (u64)chunk->buddy_mem->icm_mr->icm_start_addr + size * chunk->seg; 775c4f9b6eSRongwei Liu } 785c4f9b6eSRongwei Liu 79f51bb517SRongwei Liu u32 mlx5dr_icm_pool_get_chunk_byte_size(struct mlx5dr_icm_chunk *chunk) 80f51bb517SRongwei Liu { 81f51bb517SRongwei Liu return mlx5dr_icm_pool_chunk_size_to_byte(chunk->size, 82f51bb517SRongwei Liu chunk->buddy_mem->pool->icm_type); 83f51bb517SRongwei Liu } 84f51bb517SRongwei Liu 85f51bb517SRongwei Liu u32 mlx5dr_icm_pool_get_chunk_num_of_entries(struct mlx5dr_icm_chunk *chunk) 86f51bb517SRongwei Liu { 87f51bb517SRongwei Liu return mlx5dr_icm_pool_chunk_size_to_entries(chunk->size); 88f51bb517SRongwei Liu } 89f51bb517SRongwei Liu 9029cf8febSAlex Vesker static struct mlx5dr_icm_mr * 91dff8e2d1SErez Shitrit dr_icm_pool_mr_create(struct mlx5dr_icm_pool *pool) 9229cf8febSAlex Vesker { 9329cf8febSAlex Vesker struct mlx5_core_dev *mdev = pool->dmn->mdev; 94dff8e2d1SErez Shitrit enum mlx5_sw_icm_type dm_type; 9529cf8febSAlex Vesker struct mlx5dr_icm_mr *icm_mr; 96dff8e2d1SErez Shitrit size_t log_align_base; 9729cf8febSAlex Vesker int err; 9829cf8febSAlex Vesker 9929cf8febSAlex Vesker icm_mr = kvzalloc(sizeof(*icm_mr), GFP_KERNEL); 10029cf8febSAlex Vesker if (!icm_mr) 10129cf8febSAlex Vesker return NULL; 10229cf8febSAlex Vesker 103a00cd878SYevgeny Kliteynik icm_mr->dmn = pool->dmn; 10429cf8febSAlex Vesker 10529cf8febSAlex Vesker icm_mr->dm.length = mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 106dff8e2d1SErez Shitrit pool->icm_type); 10729cf8febSAlex Vesker 108dff8e2d1SErez Shitrit if (pool->icm_type == DR_ICM_TYPE_STE) { 109dff8e2d1SErez Shitrit dm_type = MLX5_SW_ICM_TYPE_STEERING; 110dff8e2d1SErez Shitrit log_align_base = ilog2(icm_mr->dm.length); 111dff8e2d1SErez Shitrit } else { 112dff8e2d1SErez Shitrit dm_type = MLX5_SW_ICM_TYPE_HEADER_MODIFY; 113dff8e2d1SErez Shitrit /* Align base is 64B */ 114dff8e2d1SErez Shitrit log_align_base = ilog2(DR_ICM_MODIFY_HDR_ALIGN_BASE); 115dff8e2d1SErez Shitrit } 116dff8e2d1SErez Shitrit icm_mr->dm.type = dm_type; 117dff8e2d1SErez Shitrit 118dff8e2d1SErez Shitrit err = mlx5_dm_sw_icm_alloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 119dff8e2d1SErez Shitrit log_align_base, 0, &icm_mr->dm.addr, 120dff8e2d1SErez Shitrit &icm_mr->dm.obj_id); 12129cf8febSAlex Vesker if (err) { 12229cf8febSAlex Vesker mlx5dr_err(pool->dmn, "Failed to allocate SW ICM memory, err (%d)\n", err); 12329cf8febSAlex Vesker goto free_icm_mr; 12429cf8febSAlex Vesker } 12529cf8febSAlex Vesker 12629cf8febSAlex Vesker /* Register device memory */ 12729cf8febSAlex Vesker err = dr_icm_create_dm_mkey(mdev, pool->dmn->pdn, 12829cf8febSAlex Vesker icm_mr->dm.length, 12929cf8febSAlex Vesker icm_mr->dm.addr, 13029cf8febSAlex Vesker MLX5_MKC_ACCESS_MODE_SW_ICM, 13129cf8febSAlex Vesker &icm_mr->mkey); 13229cf8febSAlex Vesker if (err) { 13329cf8febSAlex Vesker mlx5dr_err(pool->dmn, "Failed to create SW ICM MKEY, err (%d)\n", err); 13429cf8febSAlex Vesker goto free_dm; 13529cf8febSAlex Vesker } 13629cf8febSAlex Vesker 13729cf8febSAlex Vesker icm_mr->icm_start_addr = icm_mr->dm.addr; 13829cf8febSAlex Vesker 139dff8e2d1SErez Shitrit if (icm_mr->icm_start_addr & (BIT(log_align_base) - 1)) { 140dff8e2d1SErez Shitrit mlx5dr_err(pool->dmn, "Failed to get Aligned ICM mem (asked: %zu)\n", 141dff8e2d1SErez Shitrit log_align_base); 142dff8e2d1SErez Shitrit goto free_mkey; 143dff8e2d1SErez Shitrit } 14429cf8febSAlex Vesker 14529cf8febSAlex Vesker return icm_mr; 14629cf8febSAlex Vesker 147dff8e2d1SErez Shitrit free_mkey: 14883fec3f1SAharon Landau mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 14929cf8febSAlex Vesker free_dm: 15029cf8febSAlex Vesker mlx5_dm_sw_icm_dealloc(mdev, icm_mr->dm.type, icm_mr->dm.length, 0, 15129cf8febSAlex Vesker icm_mr->dm.addr, icm_mr->dm.obj_id); 15229cf8febSAlex Vesker free_icm_mr: 15329cf8febSAlex Vesker kvfree(icm_mr); 15429cf8febSAlex Vesker return NULL; 15529cf8febSAlex Vesker } 15629cf8febSAlex Vesker 15729cf8febSAlex Vesker static void dr_icm_pool_mr_destroy(struct mlx5dr_icm_mr *icm_mr) 15829cf8febSAlex Vesker { 159a00cd878SYevgeny Kliteynik struct mlx5_core_dev *mdev = icm_mr->dmn->mdev; 16029cf8febSAlex Vesker struct mlx5dr_icm_dm *dm = &icm_mr->dm; 16129cf8febSAlex Vesker 16283fec3f1SAharon Landau mlx5_core_destroy_mkey(mdev, icm_mr->mkey); 16329cf8febSAlex Vesker mlx5_dm_sw_icm_dealloc(mdev, dm->type, dm->length, 0, 16429cf8febSAlex Vesker dm->addr, dm->obj_id); 16529cf8febSAlex Vesker kvfree(icm_mr); 16629cf8febSAlex Vesker } 16729cf8febSAlex Vesker 168e5b2bc30SYevgeny Kliteynik static int dr_icm_buddy_get_ste_size(struct mlx5dr_icm_buddy_mem *buddy) 16929cf8febSAlex Vesker { 170e5b2bc30SYevgeny Kliteynik /* We support only one type of STE size, both for ConnectX-5 and later 171e5b2bc30SYevgeny Kliteynik * devices. Once the support for match STE which has a larger tag is 172e5b2bc30SYevgeny Kliteynik * added (32B instead of 16B), the STE size for devices later than 173e5b2bc30SYevgeny Kliteynik * ConnectX-5 needs to account for that. 174e5b2bc30SYevgeny Kliteynik */ 175e5b2bc30SYevgeny Kliteynik return DR_STE_SIZE_REDUCED; 176e5b2bc30SYevgeny Kliteynik } 17729cf8febSAlex Vesker 178e5b2bc30SYevgeny Kliteynik static void dr_icm_chunk_ste_init(struct mlx5dr_icm_chunk *chunk, int offset) 179e5b2bc30SYevgeny Kliteynik { 180*06ab4a40SYevgeny Kliteynik int num_of_entries = mlx5dr_icm_pool_get_chunk_num_of_entries(chunk); 181e5b2bc30SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 182*06ab4a40SYevgeny Kliteynik int ste_size = dr_icm_buddy_get_ste_size(buddy); 183e5b2bc30SYevgeny Kliteynik int index = offset / DR_STE_SIZE; 18429cf8febSAlex Vesker 185e5b2bc30SYevgeny Kliteynik chunk->ste_arr = &buddy->ste_arr[index]; 186e5b2bc30SYevgeny Kliteynik chunk->miss_list = &buddy->miss_list[index]; 187*06ab4a40SYevgeny Kliteynik chunk->hw_ste_arr = buddy->hw_ste_arr + index * ste_size; 18829cf8febSAlex Vesker 189*06ab4a40SYevgeny Kliteynik memset(chunk->hw_ste_arr, 0, num_of_entries * ste_size); 190e5b2bc30SYevgeny Kliteynik memset(chunk->ste_arr, 0, 191f51bb517SRongwei Liu num_of_entries * sizeof(chunk->ste_arr[0])); 19229cf8febSAlex Vesker } 19329cf8febSAlex Vesker 194d277b55fSYevgeny Kliteynik static void dr_icm_chunk_destroy(struct mlx5dr_icm_chunk *chunk) 19529cf8febSAlex Vesker { 196d277b55fSYevgeny Kliteynik chunk->buddy_mem->used_memory -= mlx5dr_icm_pool_get_chunk_byte_size(chunk); 19729cf8febSAlex Vesker list_del(&chunk->chunk_list); 19829cf8febSAlex Vesker 19929cf8febSAlex Vesker kvfree(chunk); 20029cf8febSAlex Vesker } 20129cf8febSAlex Vesker 202e5b2bc30SYevgeny Kliteynik static int dr_icm_buddy_init_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 203e5b2bc30SYevgeny Kliteynik { 204e5b2bc30SYevgeny Kliteynik int num_of_entries = 205e5b2bc30SYevgeny Kliteynik mlx5dr_icm_pool_chunk_size_to_entries(buddy->pool->max_log_chunk_sz); 206e5b2bc30SYevgeny Kliteynik 207e5b2bc30SYevgeny Kliteynik buddy->ste_arr = kvcalloc(num_of_entries, 208e5b2bc30SYevgeny Kliteynik sizeof(struct mlx5dr_ste), GFP_KERNEL); 209e5b2bc30SYevgeny Kliteynik if (!buddy->ste_arr) 210e5b2bc30SYevgeny Kliteynik return -ENOMEM; 211e5b2bc30SYevgeny Kliteynik 212e5b2bc30SYevgeny Kliteynik /* Preallocate full STE size on non-ConnectX-5 devices since 213e5b2bc30SYevgeny Kliteynik * we need to support both full and reduced with the same cache. 214e5b2bc30SYevgeny Kliteynik */ 215e5b2bc30SYevgeny Kliteynik buddy->hw_ste_arr = kvcalloc(num_of_entries, 216e5b2bc30SYevgeny Kliteynik dr_icm_buddy_get_ste_size(buddy), GFP_KERNEL); 217e5b2bc30SYevgeny Kliteynik if (!buddy->hw_ste_arr) 218e5b2bc30SYevgeny Kliteynik goto free_ste_arr; 219e5b2bc30SYevgeny Kliteynik 220e5b2bc30SYevgeny Kliteynik buddy->miss_list = kvmalloc(num_of_entries * sizeof(struct list_head), GFP_KERNEL); 221e5b2bc30SYevgeny Kliteynik if (!buddy->miss_list) 222e5b2bc30SYevgeny Kliteynik goto free_hw_ste_arr; 223e5b2bc30SYevgeny Kliteynik 224e5b2bc30SYevgeny Kliteynik return 0; 225e5b2bc30SYevgeny Kliteynik 226e5b2bc30SYevgeny Kliteynik free_hw_ste_arr: 227e5b2bc30SYevgeny Kliteynik kvfree(buddy->hw_ste_arr); 228e5b2bc30SYevgeny Kliteynik free_ste_arr: 229e5b2bc30SYevgeny Kliteynik kvfree(buddy->ste_arr); 230e5b2bc30SYevgeny Kliteynik return -ENOMEM; 231e5b2bc30SYevgeny Kliteynik } 232e5b2bc30SYevgeny Kliteynik 233e5b2bc30SYevgeny Kliteynik static void dr_icm_buddy_cleanup_ste_cache(struct mlx5dr_icm_buddy_mem *buddy) 234e5b2bc30SYevgeny Kliteynik { 235e5b2bc30SYevgeny Kliteynik kvfree(buddy->ste_arr); 236e5b2bc30SYevgeny Kliteynik kvfree(buddy->hw_ste_arr); 237e5b2bc30SYevgeny Kliteynik kvfree(buddy->miss_list); 238e5b2bc30SYevgeny Kliteynik } 239e5b2bc30SYevgeny Kliteynik 240a00cd878SYevgeny Kliteynik static int dr_icm_buddy_create(struct mlx5dr_icm_pool *pool) 24129cf8febSAlex Vesker { 242a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy; 243a00cd878SYevgeny Kliteynik struct mlx5dr_icm_mr *icm_mr; 24429cf8febSAlex Vesker 245a00cd878SYevgeny Kliteynik icm_mr = dr_icm_pool_mr_create(pool); 246a00cd878SYevgeny Kliteynik if (!icm_mr) 247a00cd878SYevgeny Kliteynik return -ENOMEM; 248a00cd878SYevgeny Kliteynik 249a00cd878SYevgeny Kliteynik buddy = kvzalloc(sizeof(*buddy), GFP_KERNEL); 250a00cd878SYevgeny Kliteynik if (!buddy) 251a00cd878SYevgeny Kliteynik goto free_mr; 252a00cd878SYevgeny Kliteynik 253a00cd878SYevgeny Kliteynik if (mlx5dr_buddy_init(buddy, pool->max_log_chunk_sz)) 254a00cd878SYevgeny Kliteynik goto err_free_buddy; 255a00cd878SYevgeny Kliteynik 256a00cd878SYevgeny Kliteynik buddy->icm_mr = icm_mr; 257a00cd878SYevgeny Kliteynik buddy->pool = pool; 258a00cd878SYevgeny Kliteynik 259e5b2bc30SYevgeny Kliteynik if (pool->icm_type == DR_ICM_TYPE_STE) { 260e5b2bc30SYevgeny Kliteynik /* Reduce allocations by preallocating and reusing the STE structures */ 261e5b2bc30SYevgeny Kliteynik if (dr_icm_buddy_init_ste_cache(buddy)) 262e5b2bc30SYevgeny Kliteynik goto err_cleanup_buddy; 263e5b2bc30SYevgeny Kliteynik } 264e5b2bc30SYevgeny Kliteynik 265a00cd878SYevgeny Kliteynik /* add it to the -start- of the list in order to search in it first */ 266a00cd878SYevgeny Kliteynik list_add(&buddy->list_node, &pool->buddy_mem_list); 267a00cd878SYevgeny Kliteynik 268a00cd878SYevgeny Kliteynik return 0; 269a00cd878SYevgeny Kliteynik 270e5b2bc30SYevgeny Kliteynik err_cleanup_buddy: 271e5b2bc30SYevgeny Kliteynik mlx5dr_buddy_cleanup(buddy); 272a00cd878SYevgeny Kliteynik err_free_buddy: 273a00cd878SYevgeny Kliteynik kvfree(buddy); 274a00cd878SYevgeny Kliteynik free_mr: 275a00cd878SYevgeny Kliteynik dr_icm_pool_mr_destroy(icm_mr); 276a00cd878SYevgeny Kliteynik return -ENOMEM; 27729cf8febSAlex Vesker } 27829cf8febSAlex Vesker 279a00cd878SYevgeny Kliteynik static void dr_icm_buddy_destroy(struct mlx5dr_icm_buddy_mem *buddy) 28029cf8febSAlex Vesker { 28129cf8febSAlex Vesker struct mlx5dr_icm_chunk *chunk, *next; 28229cf8febSAlex Vesker 283a00cd878SYevgeny Kliteynik list_for_each_entry_safe(chunk, next, &buddy->hot_list, chunk_list) 284d277b55fSYevgeny Kliteynik dr_icm_chunk_destroy(chunk); 28529cf8febSAlex Vesker 286a00cd878SYevgeny Kliteynik list_for_each_entry_safe(chunk, next, &buddy->used_list, chunk_list) 287d277b55fSYevgeny Kliteynik dr_icm_chunk_destroy(chunk); 288a00cd878SYevgeny Kliteynik 289a00cd878SYevgeny Kliteynik dr_icm_pool_mr_destroy(buddy->icm_mr); 290a00cd878SYevgeny Kliteynik 291a00cd878SYevgeny Kliteynik mlx5dr_buddy_cleanup(buddy); 292a00cd878SYevgeny Kliteynik 293e5b2bc30SYevgeny Kliteynik if (buddy->pool->icm_type == DR_ICM_TYPE_STE) 294e5b2bc30SYevgeny Kliteynik dr_icm_buddy_cleanup_ste_cache(buddy); 295e5b2bc30SYevgeny Kliteynik 296a00cd878SYevgeny Kliteynik kvfree(buddy); 29729cf8febSAlex Vesker } 29829cf8febSAlex Vesker 299a00cd878SYevgeny Kliteynik static struct mlx5dr_icm_chunk * 300a00cd878SYevgeny Kliteynik dr_icm_chunk_create(struct mlx5dr_icm_pool *pool, 301a00cd878SYevgeny Kliteynik enum mlx5dr_icm_chunk_size chunk_size, 302a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy_mem_pool, 303a00cd878SYevgeny Kliteynik unsigned int seg) 30429cf8febSAlex Vesker { 305a00cd878SYevgeny Kliteynik struct mlx5dr_icm_chunk *chunk; 306a00cd878SYevgeny Kliteynik int offset; 30729cf8febSAlex Vesker 308a00cd878SYevgeny Kliteynik chunk = kvzalloc(sizeof(*chunk), GFP_KERNEL); 309a00cd878SYevgeny Kliteynik if (!chunk) 310a00cd878SYevgeny Kliteynik return NULL; 31129cf8febSAlex Vesker 312a00cd878SYevgeny Kliteynik offset = mlx5dr_icm_pool_dm_type_to_entry_size(pool->icm_type) * seg; 313a00cd878SYevgeny Kliteynik 314a00cd878SYevgeny Kliteynik chunk->seg = seg; 315f51bb517SRongwei Liu chunk->size = chunk_size; 316e5b2bc30SYevgeny Kliteynik chunk->buddy_mem = buddy_mem_pool; 317a00cd878SYevgeny Kliteynik 318e5b2bc30SYevgeny Kliteynik if (pool->icm_type == DR_ICM_TYPE_STE) 319e5b2bc30SYevgeny Kliteynik dr_icm_chunk_ste_init(chunk, offset); 32029cf8febSAlex Vesker 321f51bb517SRongwei Liu buddy_mem_pool->used_memory += mlx5dr_icm_pool_get_chunk_byte_size(chunk); 322a00cd878SYevgeny Kliteynik INIT_LIST_HEAD(&chunk->chunk_list); 323a00cd878SYevgeny Kliteynik 324a00cd878SYevgeny Kliteynik /* chunk now is part of the used_list */ 325a00cd878SYevgeny Kliteynik list_add_tail(&chunk->chunk_list, &buddy_mem_pool->used_list); 326a00cd878SYevgeny Kliteynik 327a00cd878SYevgeny Kliteynik return chunk; 328a00cd878SYevgeny Kliteynik } 329a00cd878SYevgeny Kliteynik 330a00cd878SYevgeny Kliteynik static bool dr_icm_pool_is_sync_required(struct mlx5dr_icm_pool *pool) 33129cf8febSAlex Vesker { 332ecd9c5cdSYevgeny Kliteynik int allow_hot_size; 33329cf8febSAlex Vesker 334ecd9c5cdSYevgeny Kliteynik /* sync when hot memory reaches half of the pool size */ 335ecd9c5cdSYevgeny Kliteynik allow_hot_size = 336ecd9c5cdSYevgeny Kliteynik mlx5dr_icm_pool_chunk_size_to_byte(pool->max_log_chunk_sz, 337ecd9c5cdSYevgeny Kliteynik pool->icm_type) / 2; 338ecd9c5cdSYevgeny Kliteynik 339ecd9c5cdSYevgeny Kliteynik return pool->hot_memory_size > allow_hot_size; 340a00cd878SYevgeny Kliteynik } 341a00cd878SYevgeny Kliteynik 342a00cd878SYevgeny Kliteynik static int dr_icm_pool_sync_all_buddy_pools(struct mlx5dr_icm_pool *pool) 34329cf8febSAlex Vesker { 344a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 345f51bb517SRongwei Liu u32 num_entries; 346a00cd878SYevgeny Kliteynik int err; 347a00cd878SYevgeny Kliteynik 348a00cd878SYevgeny Kliteynik err = mlx5dr_cmd_sync_steering(pool->dmn->mdev); 349a00cd878SYevgeny Kliteynik if (err) { 350a00cd878SYevgeny Kliteynik mlx5dr_err(pool->dmn, "Failed to sync to HW (err: %d)\n", err); 351a00cd878SYevgeny Kliteynik return err; 35229cf8febSAlex Vesker } 35329cf8febSAlex Vesker 354a00cd878SYevgeny Kliteynik list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) { 355a00cd878SYevgeny Kliteynik struct mlx5dr_icm_chunk *chunk, *tmp_chunk; 356a00cd878SYevgeny Kliteynik 357a00cd878SYevgeny Kliteynik list_for_each_entry_safe(chunk, tmp_chunk, &buddy->hot_list, chunk_list) { 358f51bb517SRongwei Liu num_entries = mlx5dr_icm_pool_get_chunk_num_of_entries(chunk); 359f51bb517SRongwei Liu mlx5dr_buddy_free_mem(buddy, chunk->seg, ilog2(num_entries)); 360f51bb517SRongwei Liu pool->hot_memory_size -= mlx5dr_icm_pool_get_chunk_byte_size(chunk); 361d277b55fSYevgeny Kliteynik dr_icm_chunk_destroy(chunk); 362a00cd878SYevgeny Kliteynik } 363284836d9SYevgeny Kliteynik 364284836d9SYevgeny Kliteynik if (!buddy->used_memory && pool->icm_type == DR_ICM_TYPE_STE) 365284836d9SYevgeny Kliteynik dr_icm_buddy_destroy(buddy); 366a00cd878SYevgeny Kliteynik } 367a00cd878SYevgeny Kliteynik 368a00cd878SYevgeny Kliteynik return 0; 369a00cd878SYevgeny Kliteynik } 370a00cd878SYevgeny Kliteynik 371a00cd878SYevgeny Kliteynik static int dr_icm_handle_buddies_get_mem(struct mlx5dr_icm_pool *pool, 372a00cd878SYevgeny Kliteynik enum mlx5dr_icm_chunk_size chunk_size, 373a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem **buddy, 374a00cd878SYevgeny Kliteynik unsigned int *seg) 37529cf8febSAlex Vesker { 376a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy_mem_pool; 377a00cd878SYevgeny Kliteynik bool new_mem = false; 378a00cd878SYevgeny Kliteynik int err; 37929cf8febSAlex Vesker 380a00cd878SYevgeny Kliteynik alloc_buddy_mem: 381a00cd878SYevgeny Kliteynik /* find the next free place from the buddy list */ 382a00cd878SYevgeny Kliteynik list_for_each_entry(buddy_mem_pool, &pool->buddy_mem_list, list_node) { 383a00cd878SYevgeny Kliteynik err = mlx5dr_buddy_alloc_mem(buddy_mem_pool, 384a00cd878SYevgeny Kliteynik chunk_size, seg); 385a00cd878SYevgeny Kliteynik if (!err) 386a00cd878SYevgeny Kliteynik goto found; 38729cf8febSAlex Vesker 388a00cd878SYevgeny Kliteynik if (WARN_ON(new_mem)) { 389a00cd878SYevgeny Kliteynik /* We have new memory pool, first in the list */ 390a00cd878SYevgeny Kliteynik mlx5dr_err(pool->dmn, 391a00cd878SYevgeny Kliteynik "No memory for order: %d\n", 392a00cd878SYevgeny Kliteynik chunk_size); 393a00cd878SYevgeny Kliteynik goto out; 39429cf8febSAlex Vesker } 39529cf8febSAlex Vesker } 39629cf8febSAlex Vesker 397a00cd878SYevgeny Kliteynik /* no more available allocators in that pool, create new */ 398a00cd878SYevgeny Kliteynik err = dr_icm_buddy_create(pool); 399a00cd878SYevgeny Kliteynik if (err) { 400a00cd878SYevgeny Kliteynik mlx5dr_err(pool->dmn, 401a00cd878SYevgeny Kliteynik "Failed creating buddy for order %d\n", 402a00cd878SYevgeny Kliteynik chunk_size); 403a00cd878SYevgeny Kliteynik goto out; 40429cf8febSAlex Vesker } 40529cf8febSAlex Vesker 406a00cd878SYevgeny Kliteynik /* mark we have new memory, first in list */ 407a00cd878SYevgeny Kliteynik new_mem = true; 408a00cd878SYevgeny Kliteynik goto alloc_buddy_mem; 40929cf8febSAlex Vesker 410a00cd878SYevgeny Kliteynik found: 411a00cd878SYevgeny Kliteynik *buddy = buddy_mem_pool; 412a00cd878SYevgeny Kliteynik out: 413a00cd878SYevgeny Kliteynik return err; 41429cf8febSAlex Vesker } 41529cf8febSAlex Vesker 41629cf8febSAlex Vesker /* Allocate an ICM chunk, each chunk holds a piece of ICM memory and 41729cf8febSAlex Vesker * also memory used for HW STE management for optimizations. 41829cf8febSAlex Vesker */ 41929cf8febSAlex Vesker struct mlx5dr_icm_chunk * 42029cf8febSAlex Vesker mlx5dr_icm_alloc_chunk(struct mlx5dr_icm_pool *pool, 42129cf8febSAlex Vesker enum mlx5dr_icm_chunk_size chunk_size) 42229cf8febSAlex Vesker { 423a00cd878SYevgeny Kliteynik struct mlx5dr_icm_chunk *chunk = NULL; 424a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy; 425a00cd878SYevgeny Kliteynik unsigned int seg; 426a00cd878SYevgeny Kliteynik int ret; 42729cf8febSAlex Vesker 42829cf8febSAlex Vesker if (chunk_size > pool->max_log_chunk_sz) 42929cf8febSAlex Vesker return NULL; 43029cf8febSAlex Vesker 431a00cd878SYevgeny Kliteynik mutex_lock(&pool->mutex); 432a00cd878SYevgeny Kliteynik /* find mem, get back the relevant buddy pool and seg in that mem */ 433a00cd878SYevgeny Kliteynik ret = dr_icm_handle_buddies_get_mem(pool, chunk_size, &buddy, &seg); 434a00cd878SYevgeny Kliteynik if (ret) 43529cf8febSAlex Vesker goto out; 43629cf8febSAlex Vesker 437a00cd878SYevgeny Kliteynik chunk = dr_icm_chunk_create(pool, chunk_size, buddy, seg); 438a00cd878SYevgeny Kliteynik if (!chunk) 439a00cd878SYevgeny Kliteynik goto out_err; 440a00cd878SYevgeny Kliteynik 441a00cd878SYevgeny Kliteynik goto out; 442a00cd878SYevgeny Kliteynik 443a00cd878SYevgeny Kliteynik out_err: 444a00cd878SYevgeny Kliteynik mlx5dr_buddy_free_mem(buddy, seg, chunk_size); 44529cf8febSAlex Vesker out: 446a00cd878SYevgeny Kliteynik mutex_unlock(&pool->mutex); 44729cf8febSAlex Vesker return chunk; 44829cf8febSAlex Vesker } 44929cf8febSAlex Vesker 45029cf8febSAlex Vesker void mlx5dr_icm_free_chunk(struct mlx5dr_icm_chunk *chunk) 45129cf8febSAlex Vesker { 452a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy = chunk->buddy_mem; 4533eb1006aSYevgeny Kliteynik struct mlx5dr_icm_pool *pool = buddy->pool; 45429cf8febSAlex Vesker 455a00cd878SYevgeny Kliteynik /* move the memory to the waiting list AKA "hot" */ 4563eb1006aSYevgeny Kliteynik mutex_lock(&pool->mutex); 457a00cd878SYevgeny Kliteynik list_move_tail(&chunk->chunk_list, &buddy->hot_list); 458f51bb517SRongwei Liu pool->hot_memory_size += mlx5dr_icm_pool_get_chunk_byte_size(chunk); 4593eb1006aSYevgeny Kliteynik 4603eb1006aSYevgeny Kliteynik /* Check if we have chunks that are waiting for sync-ste */ 4613eb1006aSYevgeny Kliteynik if (dr_icm_pool_is_sync_required(pool)) 4623eb1006aSYevgeny Kliteynik dr_icm_pool_sync_all_buddy_pools(pool); 4633eb1006aSYevgeny Kliteynik 4643eb1006aSYevgeny Kliteynik mutex_unlock(&pool->mutex); 46529cf8febSAlex Vesker } 46629cf8febSAlex Vesker 46729cf8febSAlex Vesker struct mlx5dr_icm_pool *mlx5dr_icm_pool_create(struct mlx5dr_domain *dmn, 46829cf8febSAlex Vesker enum mlx5dr_icm_type icm_type) 46929cf8febSAlex Vesker { 47029cf8febSAlex Vesker enum mlx5dr_icm_chunk_size max_log_chunk_sz; 47129cf8febSAlex Vesker struct mlx5dr_icm_pool *pool; 47229cf8febSAlex Vesker 47329cf8febSAlex Vesker if (icm_type == DR_ICM_TYPE_STE) 47429cf8febSAlex Vesker max_log_chunk_sz = dmn->info.max_log_sw_icm_sz; 47529cf8febSAlex Vesker else 47629cf8febSAlex Vesker max_log_chunk_sz = dmn->info.max_log_action_icm_sz; 47729cf8febSAlex Vesker 47829cf8febSAlex Vesker pool = kvzalloc(sizeof(*pool), GFP_KERNEL); 47929cf8febSAlex Vesker if (!pool) 48029cf8febSAlex Vesker return NULL; 48129cf8febSAlex Vesker 48229cf8febSAlex Vesker pool->dmn = dmn; 48329cf8febSAlex Vesker pool->icm_type = icm_type; 48429cf8febSAlex Vesker pool->max_log_chunk_sz = max_log_chunk_sz; 48529cf8febSAlex Vesker 486a00cd878SYevgeny Kliteynik INIT_LIST_HEAD(&pool->buddy_mem_list); 48729cf8febSAlex Vesker 488a00cd878SYevgeny Kliteynik mutex_init(&pool->mutex); 48929cf8febSAlex Vesker 49029cf8febSAlex Vesker return pool; 49129cf8febSAlex Vesker } 49229cf8febSAlex Vesker 49329cf8febSAlex Vesker void mlx5dr_icm_pool_destroy(struct mlx5dr_icm_pool *pool) 49429cf8febSAlex Vesker { 495a00cd878SYevgeny Kliteynik struct mlx5dr_icm_buddy_mem *buddy, *tmp_buddy; 49629cf8febSAlex Vesker 497a00cd878SYevgeny Kliteynik list_for_each_entry_safe(buddy, tmp_buddy, &pool->buddy_mem_list, list_node) 498a00cd878SYevgeny Kliteynik dr_icm_buddy_destroy(buddy); 49929cf8febSAlex Vesker 500a00cd878SYevgeny Kliteynik mutex_destroy(&pool->mutex); 50129cf8febSAlex Vesker kvfree(pool); 50229cf8febSAlex Vesker } 503