1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "osdep.h" 4 #include "hmc.h" 5 #include "defs.h" 6 #include "type.h" 7 #include "protos.h" 8 #include "pble.h" 9 10 static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc); 11 12 /** 13 * irdma_destroy_pble_prm - destroy prm during module unload 14 * @pble_rsrc: pble resources 15 */ irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc * pble_rsrc)16 void irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) 17 { 18 struct irdma_chunk *chunk; 19 struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo; 20 21 while (!list_empty(&pinfo->clist)) { 22 chunk = (struct irdma_chunk *) pinfo->clist.next; 23 list_del(&chunk->list); 24 if (chunk->type == PBLE_SD_PAGED) 25 irdma_pble_free_paged_mem(chunk); 26 bitmap_free(chunk->bitmapbuf); 27 kfree(chunk->chunkmem.va); 28 } 29 } 30 31 /** 32 * irdma_hmc_init_pble - Initialize pble resources during module load 33 * @dev: irdma_sc_dev struct 34 * @pble_rsrc: pble resources 35 */ irdma_hmc_init_pble(struct irdma_sc_dev * dev,struct irdma_hmc_pble_rsrc * pble_rsrc)36 int irdma_hmc_init_pble(struct irdma_sc_dev *dev, 37 struct irdma_hmc_pble_rsrc *pble_rsrc) 38 { 39 struct irdma_hmc_info *hmc_info; 40 u32 fpm_idx = 0; 41 int status = 0; 42 43 hmc_info = dev->hmc_info; 44 pble_rsrc->dev = dev; 45 pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base; 46 /* Start pble' on 4k boundary */ 47 if (pble_rsrc->fpm_base_addr & 0xfff) 48 fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3; 49 pble_rsrc->unallocated_pble = 50 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx; 51 pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3); 52 pble_rsrc->pinfo.pble_shift = PBLE_SHIFT; 53 54 mutex_init(&pble_rsrc->pble_mutex_lock); 55 56 spin_lock_init(&pble_rsrc->pinfo.prm_lock); 57 INIT_LIST_HEAD(&pble_rsrc->pinfo.clist); 58 if (add_pble_prm(pble_rsrc)) { 59 irdma_destroy_pble_prm(pble_rsrc); 60 status = -ENOMEM; 61 } 62 63 return status; 64 } 65 66 /** 67 * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address 68 * @pble_rsrc: structure containing fpm address 69 * @idx: where to return indexes 70 */ get_sd_pd_idx(struct irdma_hmc_pble_rsrc * pble_rsrc,struct sd_pd_idx * idx)71 static void get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, 72 struct sd_pd_idx *idx) 73 { 74 idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE; 75 idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE); 76 idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD); 77 } 78 79 /** 80 * add_sd_direct - add sd direct for pble 81 * @pble_rsrc: pble resource ptr 82 * @info: page info for sd 83 */ add_sd_direct(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_add_page_info * info)84 static int add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc, 85 struct irdma_add_page_info *info) 86 { 87 struct irdma_sc_dev *dev = pble_rsrc->dev; 88 int ret_code = 0; 89 struct sd_pd_idx *idx = &info->idx; 90 struct irdma_chunk *chunk = info->chunk; 91 struct irdma_hmc_info *hmc_info = info->hmc_info; 92 struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; 93 u32 offset = 0; 94 95 if (!sd_entry->valid) { 96 ret_code = irdma_add_sd_table_entry(dev->hw, hmc_info, 97 info->idx.sd_idx, 98 IRDMA_SD_TYPE_DIRECT, 99 IRDMA_HMC_DIRECT_BP_SIZE); 100 if (ret_code) 101 return ret_code; 102 103 chunk->type = PBLE_SD_CONTIGOUS; 104 } 105 106 offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT; 107 chunk->size = info->pages << HMC_PAGED_BP_SHIFT; 108 chunk->vaddr = sd_entry->u.bp.addr.va + offset; 109 chunk->fpm_addr = pble_rsrc->next_fpm_addr; 110 ibdev_dbg(to_ibdev(dev), 111 "PBLE: chunk_size[%lld] = 0x%llx vaddr=0x%pK fpm_addr = %llx\n", 112 chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr); 113 114 return 0; 115 } 116 117 /** 118 * fpm_to_idx - given fpm address, get pble index 119 * @pble_rsrc: pble resource management 120 * @addr: fpm address for index 121 */ fpm_to_idx(struct irdma_hmc_pble_rsrc * pble_rsrc,u64 addr)122 static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr) 123 { 124 u64 idx; 125 126 idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3; 127 128 return (u32)idx; 129 } 130 131 /** 132 * add_bp_pages - add backing pages for sd 133 * @pble_rsrc: pble resource management 134 * @info: page info for sd 135 */ add_bp_pages(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_add_page_info * info)136 static int add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc, 137 struct irdma_add_page_info *info) 138 { 139 struct irdma_sc_dev *dev = pble_rsrc->dev; 140 u8 *addr; 141 struct irdma_dma_mem mem; 142 struct irdma_hmc_pd_entry *pd_entry; 143 struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; 144 struct irdma_hmc_info *hmc_info = info->hmc_info; 145 struct irdma_chunk *chunk = info->chunk; 146 int status = 0; 147 u32 rel_pd_idx = info->idx.rel_pd_idx; 148 u32 pd_idx = info->idx.pd_idx; 149 u32 i; 150 151 if (irdma_pble_get_paged_mem(chunk, info->pages)) 152 return -ENOMEM; 153 154 status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx, 155 IRDMA_SD_TYPE_PAGED, 156 IRDMA_HMC_DIRECT_BP_SIZE); 157 if (status) 158 goto error; 159 160 addr = chunk->vaddr; 161 for (i = 0; i < info->pages; i++) { 162 mem.pa = (u64)chunk->dmainfo.dmaaddrs[i]; 163 mem.size = 4096; 164 mem.va = addr; 165 pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++]; 166 if (!pd_entry->valid) { 167 status = irdma_add_pd_table_entry(dev, hmc_info, 168 pd_idx++, &mem); 169 if (status) 170 goto error; 171 172 addr += 4096; 173 } 174 } 175 176 chunk->fpm_addr = pble_rsrc->next_fpm_addr; 177 return 0; 178 179 error: 180 irdma_pble_free_paged_mem(chunk); 181 182 return status; 183 } 184 185 /** 186 * irdma_get_type - add a sd entry type for sd 187 * @dev: irdma_sc_dev struct 188 * @idx: index of sd 189 * @pages: pages in the sd 190 */ irdma_get_type(struct irdma_sc_dev * dev,struct sd_pd_idx * idx,u32 pages)191 static enum irdma_sd_entry_type irdma_get_type(struct irdma_sc_dev *dev, 192 struct sd_pd_idx *idx, u32 pages) 193 { 194 enum irdma_sd_entry_type sd_entry_type; 195 196 sd_entry_type = !idx->rel_pd_idx && pages == IRDMA_HMC_PD_CNT_IN_SD ? 197 IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED; 198 return sd_entry_type; 199 } 200 201 /** 202 * add_pble_prm - add a sd entry for pble resoure 203 * @pble_rsrc: pble resource management 204 */ add_pble_prm(struct irdma_hmc_pble_rsrc * pble_rsrc)205 static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) 206 { 207 struct irdma_sc_dev *dev = pble_rsrc->dev; 208 struct irdma_hmc_sd_entry *sd_entry; 209 struct irdma_hmc_info *hmc_info; 210 struct irdma_chunk *chunk; 211 struct irdma_add_page_info info; 212 struct sd_pd_idx *idx = &info.idx; 213 int ret_code = 0; 214 enum irdma_sd_entry_type sd_entry_type; 215 u64 sd_reg_val = 0; 216 struct irdma_virt_mem chunkmem; 217 u32 pages; 218 219 if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE) 220 return -ENOMEM; 221 222 if (pble_rsrc->next_fpm_addr & 0xfff) 223 return -EINVAL; 224 225 chunkmem.size = sizeof(*chunk); 226 chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL); 227 if (!chunkmem.va) 228 return -ENOMEM; 229 230 chunk = chunkmem.va; 231 chunk->chunkmem = chunkmem; 232 hmc_info = dev->hmc_info; 233 chunk->dev = dev; 234 chunk->fpm_addr = pble_rsrc->next_fpm_addr; 235 get_sd_pd_idx(pble_rsrc, idx); 236 sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx]; 237 pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) : 238 IRDMA_HMC_PD_CNT_IN_SD; 239 pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT); 240 info.chunk = chunk; 241 info.hmc_info = hmc_info; 242 info.pages = pages; 243 info.sd_entry = sd_entry; 244 if (!sd_entry->valid) 245 sd_entry_type = irdma_get_type(dev, idx, pages); 246 else 247 sd_entry_type = sd_entry->entry_type; 248 249 ibdev_dbg(to_ibdev(dev), 250 "PBLE: pages = %d, unallocated_pble[%d] current_fpm_addr = %llx\n", 251 pages, pble_rsrc->unallocated_pble, 252 pble_rsrc->next_fpm_addr); 253 ibdev_dbg(to_ibdev(dev), "PBLE: sd_entry_type = %d\n", sd_entry_type); 254 if (sd_entry_type == IRDMA_SD_TYPE_DIRECT) 255 ret_code = add_sd_direct(pble_rsrc, &info); 256 257 if (ret_code) 258 sd_entry_type = IRDMA_SD_TYPE_PAGED; 259 else 260 pble_rsrc->stats_direct_sds++; 261 262 if (sd_entry_type == IRDMA_SD_TYPE_PAGED) { 263 ret_code = add_bp_pages(pble_rsrc, &info); 264 if (ret_code) 265 goto error; 266 else 267 pble_rsrc->stats_paged_sds++; 268 } 269 270 ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk); 271 if (ret_code) 272 goto error; 273 274 pble_rsrc->next_fpm_addr += chunk->size; 275 ibdev_dbg(to_ibdev(dev), 276 "PBLE: next_fpm_addr = %llx chunk_size[%llu] = 0x%llx\n", 277 pble_rsrc->next_fpm_addr, chunk->size, chunk->size); 278 pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3); 279 sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ? 280 sd_entry->u.pd_table.pd_page_addr.pa : 281 sd_entry->u.bp.addr.pa; 282 283 if (!sd_entry->valid) { 284 ret_code = irdma_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val, 285 idx->sd_idx, sd_entry->entry_type, true); 286 if (ret_code) 287 goto error; 288 } 289 290 list_add(&chunk->list, &pble_rsrc->pinfo.clist); 291 sd_entry->valid = true; 292 return 0; 293 294 error: 295 bitmap_free(chunk->bitmapbuf); 296 kfree(chunk->chunkmem.va); 297 298 return ret_code; 299 } 300 301 /** 302 * free_lvl2 - fee level 2 pble 303 * @pble_rsrc: pble resource management 304 * @palloc: level 2 pble allocation 305 */ free_lvl2(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc)306 static void free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc, 307 struct irdma_pble_alloc *palloc) 308 { 309 u32 i; 310 struct irdma_pble_level2 *lvl2 = &palloc->level2; 311 struct irdma_pble_info *root = &lvl2->root; 312 struct irdma_pble_info *leaf = lvl2->leaf; 313 314 for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 315 if (leaf->addr) 316 irdma_prm_return_pbles(&pble_rsrc->pinfo, 317 &leaf->chunkinfo); 318 else 319 break; 320 } 321 322 if (root->addr) 323 irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo); 324 325 kfree(lvl2->leafmem.va); 326 lvl2->leaf = NULL; 327 } 328 329 /** 330 * get_lvl2_pble - get level 2 pble resource 331 * @pble_rsrc: pble resource management 332 * @palloc: level 2 pble allocation 333 */ get_lvl2_pble(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc)334 static int get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 335 struct irdma_pble_alloc *palloc) 336 { 337 u32 lf4k, lflast, total, i; 338 u32 pblcnt = PBLE_PER_PAGE; 339 u64 *addr; 340 struct irdma_pble_level2 *lvl2 = &palloc->level2; 341 struct irdma_pble_info *root = &lvl2->root; 342 struct irdma_pble_info *leaf; 343 int ret_code; 344 u64 fpm_addr; 345 346 /* number of full 512 (4K) leafs) */ 347 lf4k = palloc->total_cnt >> 9; 348 lflast = palloc->total_cnt % PBLE_PER_PAGE; 349 total = (lflast == 0) ? lf4k : lf4k + 1; 350 lvl2->leaf_cnt = total; 351 352 lvl2->leafmem.size = (sizeof(*leaf) * total); 353 lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL); 354 if (!lvl2->leafmem.va) 355 return -ENOMEM; 356 357 lvl2->leaf = lvl2->leafmem.va; 358 leaf = lvl2->leaf; 359 ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo, 360 total << 3, &root->addr, &fpm_addr); 361 if (ret_code) { 362 kfree(lvl2->leafmem.va); 363 lvl2->leaf = NULL; 364 return -ENOMEM; 365 } 366 367 root->idx = fpm_to_idx(pble_rsrc, fpm_addr); 368 root->cnt = total; 369 addr = root->addr; 370 for (i = 0; i < total; i++, leaf++) { 371 pblcnt = (lflast && ((i + 1) == total)) ? 372 lflast : PBLE_PER_PAGE; 373 ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, 374 &leaf->chunkinfo, pblcnt << 3, 375 &leaf->addr, &fpm_addr); 376 if (ret_code) 377 goto error; 378 379 leaf->idx = fpm_to_idx(pble_rsrc, fpm_addr); 380 381 leaf->cnt = pblcnt; 382 *addr = (u64)leaf->idx; 383 addr++; 384 } 385 386 palloc->level = PBLE_LEVEL_2; 387 pble_rsrc->stats_lvl2++; 388 return 0; 389 390 error: 391 free_lvl2(pble_rsrc, palloc); 392 393 return -ENOMEM; 394 } 395 396 /** 397 * get_lvl1_pble - get level 1 pble resource 398 * @pble_rsrc: pble resource management 399 * @palloc: level 1 pble allocation 400 */ get_lvl1_pble(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc)401 static int get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 402 struct irdma_pble_alloc *palloc) 403 { 404 int ret_code; 405 u64 fpm_addr; 406 struct irdma_pble_info *lvl1 = &palloc->level1; 407 408 ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &lvl1->chunkinfo, 409 palloc->total_cnt << 3, &lvl1->addr, 410 &fpm_addr); 411 if (ret_code) 412 return -ENOMEM; 413 414 palloc->level = PBLE_LEVEL_1; 415 lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr); 416 lvl1->cnt = palloc->total_cnt; 417 pble_rsrc->stats_lvl1++; 418 419 return 0; 420 } 421 422 /** 423 * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine 424 * @pble_rsrc: pble resources 425 * @palloc: contains all inforamtion regarding pble (idx + pble addr) 426 * @lvl: Bitmask for requested pble level 427 */ get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc,u8 lvl)428 static int get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 429 struct irdma_pble_alloc *palloc, u8 lvl) 430 { 431 int status = 0; 432 433 status = get_lvl1_pble(pble_rsrc, palloc); 434 if (!status || lvl == PBLE_LEVEL_1 || palloc->total_cnt <= PBLE_PER_PAGE) 435 return status; 436 437 status = get_lvl2_pble(pble_rsrc, palloc); 438 439 return status; 440 } 441 442 /** 443 * irdma_get_pble - allocate pbles from the prm 444 * @pble_rsrc: pble resources 445 * @palloc: contains all inforamtion regarding pble (idx + pble addr) 446 * @pble_cnt: #of pbles requested 447 * @lvl: requested pble level mask 448 */ irdma_get_pble(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc,u32 pble_cnt,u8 lvl)449 int irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 450 struct irdma_pble_alloc *palloc, u32 pble_cnt, 451 u8 lvl) 452 { 453 int status = 0; 454 int max_sds = 0; 455 int i; 456 457 palloc->total_cnt = pble_cnt; 458 palloc->level = PBLE_LEVEL_0; 459 460 mutex_lock(&pble_rsrc->pble_mutex_lock); 461 462 /*check first to see if we can get pble's without acquiring 463 * additional sd's 464 */ 465 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); 466 if (!status) 467 goto exit; 468 469 max_sds = (palloc->total_cnt >> 18) + 1; 470 for (i = 0; i < max_sds; i++) { 471 status = add_pble_prm(pble_rsrc); 472 if (status) 473 break; 474 475 status = get_lvl1_lvl2_pble(pble_rsrc, palloc, lvl); 476 /* if level1_only, only go through it once */ 477 if (!status || lvl) 478 break; 479 } 480 481 exit: 482 if (!status) { 483 pble_rsrc->allocdpbles += pble_cnt; 484 pble_rsrc->stats_alloc_ok++; 485 } else { 486 pble_rsrc->stats_alloc_fail++; 487 } 488 mutex_unlock(&pble_rsrc->pble_mutex_lock); 489 490 return status; 491 } 492 493 /** 494 * irdma_free_pble - put pbles back into prm 495 * @pble_rsrc: pble resources 496 * @palloc: contains all information regarding pble resource being freed 497 */ irdma_free_pble(struct irdma_hmc_pble_rsrc * pble_rsrc,struct irdma_pble_alloc * palloc)498 void irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 499 struct irdma_pble_alloc *palloc) 500 { 501 pble_rsrc->freedpbles += palloc->total_cnt; 502 503 if (palloc->level == PBLE_LEVEL_2) 504 free_lvl2(pble_rsrc, palloc); 505 else 506 irdma_prm_return_pbles(&pble_rsrc->pinfo, 507 &palloc->level1.chunkinfo); 508 pble_rsrc->stats_alloc_freed++; 509 } 510