1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 2 /* Copyright (c) 2015 - 2021 Intel Corporation */ 3 #include "osdep.h" 4 #include "hmc.h" 5 #include "defs.h" 6 #include "type.h" 7 #include "protos.h" 8 9 /** 10 * irdma_find_sd_index_limit - finds segment descriptor index limit 11 * @hmc_info: pointer to the HMC configuration information structure 12 * @type: type of HMC resources we're searching 13 * @idx: starting index for the object 14 * @cnt: number of objects we're trying to create 15 * @sd_idx: pointer to return index of the segment descriptor in question 16 * @sd_limit: pointer to return the maximum number of segment descriptors 17 * 18 * This function calculates the segment descriptor index and index limit 19 * for the resource defined by irdma_hmc_rsrc_type. 20 */ 21 22 static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type, 23 u32 idx, u32 cnt, u32 *sd_idx, 24 u32 *sd_limit) 25 { 26 u64 fpm_addr, fpm_limit; 27 28 fpm_addr = hmc_info->hmc_obj[(type)].base + 29 hmc_info->hmc_obj[type].size * idx; 30 fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt; 31 *sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE); 32 *sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE); 33 *sd_limit += 1; 34 } 35 36 /** 37 * irdma_find_pd_index_limit - finds page descriptor index limit 38 * @hmc_info: pointer to the HMC configuration information struct 39 * @type: HMC resource type we're examining 40 * @idx: starting index for the object 41 * @cnt: number of objects we're trying to create 42 * @pd_idx: pointer to return page descriptor index 43 * @pd_limit: pointer to return page descriptor index limit 44 * 45 * Calculates the page descriptor index and index limit for the resource 46 * defined by irdma_hmc_rsrc_type. 47 */ 48 49 static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type, 50 u32 idx, u32 cnt, u32 *pd_idx, 51 u32 *pd_limit) 52 { 53 u64 fpm_adr, fpm_limit; 54 55 fpm_adr = hmc_info->hmc_obj[type].base + 56 hmc_info->hmc_obj[type].size * idx; 57 fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt); 58 *pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE); 59 *pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE); 60 *pd_limit += 1; 61 } 62 63 /** 64 * irdma_set_sd_entry - setup entry for sd programming 65 * @pa: physical addr 66 * @idx: sd index 67 * @type: paged or direct sd 68 * @entry: sd entry ptr 69 */ 70 static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type, 71 struct irdma_update_sd_entry *entry) 72 { 73 entry->data = pa | 74 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) | 75 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE, 76 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) | 77 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1); 78 79 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15); 80 } 81 82 /** 83 * irdma_clr_sd_entry - setup entry for sd clear 84 * @idx: sd index 85 * @type: paged or direct sd 86 * @entry: sd entry ptr 87 */ 88 static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type, 89 struct irdma_update_sd_entry *entry) 90 { 91 entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) | 92 FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE, 93 type == IRDMA_SD_TYPE_PAGED ? 0 : 1); 94 95 entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15); 96 } 97 98 /** 99 * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF 100 * @dev: pointer to our device struct 101 * @sd_idx: segment descriptor index 102 * @pd_idx: page descriptor index 103 */ 104 static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx, 105 u32 pd_idx) 106 { 107 u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) | 108 FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) | 109 FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx); 110 111 writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]); 112 } 113 114 /** 115 * irdma_hmc_sd_one - setup 1 sd entry for cqp 116 * @dev: pointer to the device structure 117 * @hmc_fn_id: hmc's function id 118 * @pa: physical addr 119 * @sd_idx: sd index 120 * @type: paged or direct sd 121 * @setsd: flag to set or clear sd 122 */ 123 int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx, 124 enum irdma_sd_entry_type type, bool setsd) 125 { 126 struct irdma_update_sds_info sdinfo; 127 128 sdinfo.cnt = 1; 129 sdinfo.hmc_fn_id = hmc_fn_id; 130 if (setsd) 131 irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry); 132 else 133 irdma_clr_sd_entry(sd_idx, type, sdinfo.entry); 134 return dev->cqp->process_cqp_sds(dev, &sdinfo); 135 } 136 137 /** 138 * irdma_hmc_sd_grp - setup group of sd entries for cqp 139 * @dev: pointer to the device structure 140 * @hmc_info: pointer to the HMC configuration information struct 141 * @sd_index: sd index 142 * @sd_cnt: number of sd entries 143 * @setsd: flag to set or clear sd 144 */ 145 static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev, 146 struct irdma_hmc_info *hmc_info, u32 sd_index, 147 u32 sd_cnt, bool setsd) 148 { 149 struct irdma_hmc_sd_entry *sd_entry; 150 struct irdma_update_sds_info sdinfo = {}; 151 u64 pa; 152 u32 i; 153 int ret_code = 0; 154 155 sdinfo.hmc_fn_id = hmc_info->hmc_fn_id; 156 for (i = sd_index; i < sd_index + sd_cnt; i++) { 157 sd_entry = &hmc_info->sd_table.sd_entry[i]; 158 if (!sd_entry || (!sd_entry->valid && setsd) || 159 (sd_entry->valid && !setsd)) 160 continue; 161 if (setsd) { 162 pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ? 163 sd_entry->u.pd_table.pd_page_addr.pa : 164 sd_entry->u.bp.addr.pa; 165 irdma_set_sd_entry(pa, i, sd_entry->entry_type, 166 &sdinfo.entry[sdinfo.cnt]); 167 } else { 168 irdma_clr_sd_entry(i, sd_entry->entry_type, 169 &sdinfo.entry[sdinfo.cnt]); 170 } 171 sdinfo.cnt++; 172 if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) { 173 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); 174 if (ret_code) { 175 ibdev_dbg(to_ibdev(dev), 176 "HMC: sd_programming failed err=%d\n", 177 ret_code); 178 return ret_code; 179 } 180 181 sdinfo.cnt = 0; 182 } 183 } 184 if (sdinfo.cnt) 185 ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo); 186 187 return ret_code; 188 } 189 190 /** 191 * irdma_hmc_finish_add_sd_reg - program sd entries for objects 192 * @dev: pointer to the device structure 193 * @info: create obj info 194 */ 195 static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev, 196 struct irdma_hmc_create_obj_info *info) 197 { 198 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) 199 return -EINVAL; 200 201 if ((info->start_idx + info->count) > 202 info->hmc_info->hmc_obj[info->rsrc_type].cnt) 203 return -EINVAL; 204 205 if (!info->add_sd_cnt) 206 return 0; 207 return irdma_hmc_sd_grp(dev, info->hmc_info, 208 info->hmc_info->sd_indexes[0], info->add_sd_cnt, 209 true); 210 } 211 212 /** 213 * irdma_sc_create_hmc_obj - allocate backing store for hmc objects 214 * @dev: pointer to the device structure 215 * @info: pointer to irdma_hmc_create_obj_info struct 216 * 217 * This will allocate memory for PDs and backing pages and populate 218 * the sd and pd entries. 219 */ 220 int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev, 221 struct irdma_hmc_create_obj_info *info) 222 { 223 struct irdma_hmc_sd_entry *sd_entry; 224 u32 sd_idx, sd_lmt; 225 u32 pd_idx = 0, pd_lmt = 0; 226 u32 pd_idx1 = 0, pd_lmt1 = 0; 227 u32 i, j; 228 bool pd_error = false; 229 int ret_code = 0; 230 231 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) 232 return -EINVAL; 233 234 if ((info->start_idx + info->count) > 235 info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 236 ibdev_dbg(to_ibdev(dev), 237 "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n", 238 info->rsrc_type, info->start_idx, info->count, 239 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 240 return -EINVAL; 241 } 242 243 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type, 244 info->start_idx, info->count, &sd_idx, 245 &sd_lmt); 246 if (sd_idx >= info->hmc_info->sd_table.sd_cnt || 247 sd_lmt > info->hmc_info->sd_table.sd_cnt) { 248 return -EINVAL; 249 } 250 251 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type, 252 info->start_idx, info->count, &pd_idx, 253 &pd_lmt); 254 255 for (j = sd_idx; j < sd_lmt; j++) { 256 ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j, 257 info->entry_type, 258 IRDMA_HMC_DIRECT_BP_SIZE); 259 if (ret_code) 260 goto exit_sd_error; 261 262 sd_entry = &info->hmc_info->sd_table.sd_entry[j]; 263 if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED && 264 (dev->hmc_info == info->hmc_info && 265 info->rsrc_type != IRDMA_HMC_IW_PBLE)) { 266 pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT)); 267 pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT); 268 for (i = pd_idx1; i < pd_lmt1; i++) { 269 /* update the pd table entry */ 270 ret_code = irdma_add_pd_table_entry(dev, 271 info->hmc_info, 272 i, NULL); 273 if (ret_code) { 274 pd_error = true; 275 break; 276 } 277 } 278 if (pd_error) { 279 while (i && (i > pd_idx1)) { 280 irdma_remove_pd_bp(dev, info->hmc_info, 281 i - 1); 282 i--; 283 } 284 } 285 } 286 if (sd_entry->valid) 287 continue; 288 289 info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j; 290 info->add_sd_cnt++; 291 sd_entry->valid = true; 292 } 293 return irdma_hmc_finish_add_sd_reg(dev, info); 294 295 exit_sd_error: 296 while (j && (j > sd_idx)) { 297 sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1]; 298 switch (sd_entry->entry_type) { 299 case IRDMA_SD_TYPE_PAGED: 300 pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT); 301 pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT)); 302 for (i = pd_idx1; i < pd_lmt1; i++) 303 irdma_prep_remove_pd_page(info->hmc_info, i); 304 break; 305 case IRDMA_SD_TYPE_DIRECT: 306 irdma_prep_remove_pd_page(info->hmc_info, (j - 1)); 307 break; 308 default: 309 ret_code = -EINVAL; 310 break; 311 } 312 j--; 313 } 314 315 return ret_code; 316 } 317 318 /** 319 * irdma_finish_del_sd_reg - delete sd entries for objects 320 * @dev: pointer to the device structure 321 * @info: dele obj info 322 * @reset: true if called before reset 323 */ 324 static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev, 325 struct irdma_hmc_del_obj_info *info, 326 bool reset) 327 { 328 struct irdma_hmc_sd_entry *sd_entry; 329 int ret_code = 0; 330 u32 i, sd_idx; 331 struct irdma_dma_mem *mem; 332 333 if (!reset) 334 ret_code = irdma_hmc_sd_grp(dev, info->hmc_info, 335 info->hmc_info->sd_indexes[0], 336 info->del_sd_cnt, false); 337 338 if (ret_code) 339 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n"); 340 for (i = 0; i < info->del_sd_cnt; i++) { 341 sd_idx = info->hmc_info->sd_indexes[i]; 342 sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx]; 343 mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ? 344 &sd_entry->u.pd_table.pd_page_addr : 345 &sd_entry->u.bp.addr; 346 347 if (!mem || !mem->va) { 348 ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n"); 349 } else { 350 dma_free_coherent(dev->hw->device, mem->size, mem->va, 351 mem->pa); 352 mem->va = NULL; 353 } 354 } 355 356 return ret_code; 357 } 358 359 /** 360 * irdma_sc_del_hmc_obj - remove pe hmc objects 361 * @dev: pointer to the device structure 362 * @info: pointer to irdma_hmc_del_obj_info struct 363 * @reset: true if called before reset 364 * 365 * This will de-populate the SDs and PDs. It frees 366 * the memory for PDS and backing storage. After this function is returned, 367 * caller should deallocate memory allocated previously for 368 * book-keeping information about PDs and backing storage. 369 */ 370 int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev, 371 struct irdma_hmc_del_obj_info *info, bool reset) 372 { 373 struct irdma_hmc_pd_table *pd_table; 374 u32 sd_idx, sd_lmt; 375 u32 pd_idx, pd_lmt, rel_pd_idx; 376 u32 i, j; 377 int ret_code = 0; 378 379 if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 380 ibdev_dbg(to_ibdev(dev), 381 "HMC: error start_idx[%04d] >= [type %04d].cnt[%04d]\n", 382 info->start_idx, info->rsrc_type, 383 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 384 return -EINVAL; 385 } 386 387 if ((info->start_idx + info->count) > 388 info->hmc_info->hmc_obj[info->rsrc_type].cnt) { 389 ibdev_dbg(to_ibdev(dev), 390 "HMC: error start_idx[%04d] + count %04d >= [type %04d].cnt[%04d]\n", 391 info->start_idx, info->count, info->rsrc_type, 392 info->hmc_info->hmc_obj[info->rsrc_type].cnt); 393 return -EINVAL; 394 } 395 396 irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type, 397 info->start_idx, info->count, &pd_idx, 398 &pd_lmt); 399 400 for (j = pd_idx; j < pd_lmt; j++) { 401 sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD; 402 403 if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid) 404 continue; 405 406 if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type != 407 IRDMA_SD_TYPE_PAGED) 408 continue; 409 410 rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD; 411 pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 412 if (pd_table->pd_entry && 413 pd_table->pd_entry[rel_pd_idx].valid) { 414 ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j); 415 if (ret_code) { 416 ibdev_dbg(to_ibdev(dev), 417 "HMC: remove_pd_bp error\n"); 418 return ret_code; 419 } 420 } 421 } 422 423 irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type, 424 info->start_idx, info->count, &sd_idx, 425 &sd_lmt); 426 if (sd_idx >= info->hmc_info->sd_table.sd_cnt || 427 sd_lmt > info->hmc_info->sd_table.sd_cnt) { 428 ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n"); 429 return -EINVAL; 430 } 431 432 for (i = sd_idx; i < sd_lmt; i++) { 433 pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table; 434 if (!info->hmc_info->sd_table.sd_entry[i].valid) 435 continue; 436 switch (info->hmc_info->sd_table.sd_entry[i].entry_type) { 437 case IRDMA_SD_TYPE_DIRECT: 438 ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i); 439 if (!ret_code) { 440 info->hmc_info->sd_indexes[info->del_sd_cnt] = 441 (u16)i; 442 info->del_sd_cnt++; 443 } 444 break; 445 case IRDMA_SD_TYPE_PAGED: 446 ret_code = irdma_prep_remove_pd_page(info->hmc_info, i); 447 if (ret_code) 448 break; 449 if (dev->hmc_info != info->hmc_info && 450 info->rsrc_type == IRDMA_HMC_IW_PBLE && 451 pd_table->pd_entry) { 452 kfree(pd_table->pd_entry_virt_mem.va); 453 pd_table->pd_entry = NULL; 454 } 455 info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i; 456 info->del_sd_cnt++; 457 break; 458 default: 459 break; 460 } 461 } 462 return irdma_finish_del_sd_reg(dev, info, reset); 463 } 464 465 /** 466 * irdma_add_sd_table_entry - Adds a segment descriptor to the table 467 * @hw: pointer to our hw struct 468 * @hmc_info: pointer to the HMC configuration information struct 469 * @sd_index: segment descriptor index to manipulate 470 * @type: what type of segment descriptor we're manipulating 471 * @direct_mode_sz: size to alloc in direct mode 472 */ 473 int irdma_add_sd_table_entry(struct irdma_hw *hw, 474 struct irdma_hmc_info *hmc_info, u32 sd_index, 475 enum irdma_sd_entry_type type, u64 direct_mode_sz) 476 { 477 struct irdma_hmc_sd_entry *sd_entry; 478 struct irdma_dma_mem dma_mem; 479 u64 alloc_len; 480 481 sd_entry = &hmc_info->sd_table.sd_entry[sd_index]; 482 if (!sd_entry->valid) { 483 if (type == IRDMA_SD_TYPE_PAGED) 484 alloc_len = IRDMA_HMC_PAGED_BP_SIZE; 485 else 486 alloc_len = direct_mode_sz; 487 488 /* allocate a 4K pd page or 2M backing page */ 489 dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT); 490 dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size, 491 &dma_mem.pa, GFP_KERNEL); 492 if (!dma_mem.va) 493 return -ENOMEM; 494 if (type == IRDMA_SD_TYPE_PAGED) { 495 struct irdma_virt_mem *vmem = 496 &sd_entry->u.pd_table.pd_entry_virt_mem; 497 498 vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512; 499 vmem->va = kzalloc(vmem->size, GFP_KERNEL); 500 if (!vmem->va) { 501 dma_free_coherent(hw->device, dma_mem.size, 502 dma_mem.va, dma_mem.pa); 503 dma_mem.va = NULL; 504 return -ENOMEM; 505 } 506 sd_entry->u.pd_table.pd_entry = vmem->va; 507 508 memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem, 509 sizeof(sd_entry->u.pd_table.pd_page_addr)); 510 } else { 511 memcpy(&sd_entry->u.bp.addr, &dma_mem, 512 sizeof(sd_entry->u.bp.addr)); 513 514 sd_entry->u.bp.sd_pd_index = sd_index; 515 } 516 517 hmc_info->sd_table.sd_entry[sd_index].entry_type = type; 518 hmc_info->sd_table.use_cnt++; 519 } 520 if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT) 521 sd_entry->u.bp.use_cnt++; 522 523 return 0; 524 } 525 526 /** 527 * irdma_add_pd_table_entry - Adds page descriptor to the specified table 528 * @dev: pointer to our device structure 529 * @hmc_info: pointer to the HMC configuration information structure 530 * @pd_index: which page descriptor index to manipulate 531 * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one. 532 * 533 * This function: 534 * 1. Initializes the pd entry 535 * 2. Adds pd_entry in the pd_table 536 * 3. Mark the entry valid in irdma_hmc_pd_entry structure 537 * 4. Initializes the pd_entry's ref count to 1 538 * assumptions: 539 * 1. The memory for pd should be pinned down, physically contiguous and 540 * aligned on 4K boundary and zeroed memory. 541 * 2. It should be 4K in size. 542 */ 543 int irdma_add_pd_table_entry(struct irdma_sc_dev *dev, 544 struct irdma_hmc_info *hmc_info, u32 pd_index, 545 struct irdma_dma_mem *rsrc_pg) 546 { 547 struct irdma_hmc_pd_table *pd_table; 548 struct irdma_hmc_pd_entry *pd_entry; 549 struct irdma_dma_mem mem; 550 struct irdma_dma_mem *page = &mem; 551 u32 sd_idx, rel_pd_idx; 552 u64 *pd_addr; 553 u64 page_desc; 554 555 if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) 556 return -EINVAL; 557 558 sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD); 559 if (hmc_info->sd_table.sd_entry[sd_idx].entry_type != 560 IRDMA_SD_TYPE_PAGED) 561 return 0; 562 563 rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD); 564 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 565 pd_entry = &pd_table->pd_entry[rel_pd_idx]; 566 if (!pd_entry->valid) { 567 if (rsrc_pg) { 568 pd_entry->rsrc_pg = true; 569 page = rsrc_pg; 570 } else { 571 page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE, 572 IRDMA_HMC_PD_BP_BUF_ALIGNMENT); 573 page->va = dma_alloc_coherent(dev->hw->device, 574 page->size, &page->pa, 575 GFP_KERNEL); 576 if (!page->va) 577 return -ENOMEM; 578 579 pd_entry->rsrc_pg = false; 580 } 581 582 memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr)); 583 pd_entry->bp.sd_pd_index = pd_index; 584 pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED; 585 page_desc = page->pa | 0x1; 586 pd_addr = pd_table->pd_page_addr.va; 587 pd_addr += rel_pd_idx; 588 memcpy(pd_addr, &page_desc, sizeof(*pd_addr)); 589 pd_entry->sd_index = sd_idx; 590 pd_entry->valid = true; 591 pd_table->use_cnt++; 592 irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx); 593 } 594 pd_entry->bp.use_cnt++; 595 596 return 0; 597 } 598 599 /** 600 * irdma_remove_pd_bp - remove a backing page from a page descriptor 601 * @dev: pointer to our HW structure 602 * @hmc_info: pointer to the HMC configuration information structure 603 * @idx: the page index 604 * 605 * This function: 606 * 1. Marks the entry in pd table (for paged address mode) or in sd table 607 * (for direct address mode) invalid. 608 * 2. Write to register PMPDINV to invalidate the backing page in FV cache 609 * 3. Decrement the ref count for the pd _entry 610 * assumptions: 611 * 1. Caller can deallocate the memory used by backing storage after this 612 * function returns. 613 */ 614 int irdma_remove_pd_bp(struct irdma_sc_dev *dev, 615 struct irdma_hmc_info *hmc_info, u32 idx) 616 { 617 struct irdma_hmc_pd_entry *pd_entry; 618 struct irdma_hmc_pd_table *pd_table; 619 struct irdma_hmc_sd_entry *sd_entry; 620 u32 sd_idx, rel_pd_idx; 621 struct irdma_dma_mem *mem; 622 u64 *pd_addr; 623 624 sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD; 625 rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD; 626 if (sd_idx >= hmc_info->sd_table.sd_cnt) 627 return -EINVAL; 628 629 sd_entry = &hmc_info->sd_table.sd_entry[sd_idx]; 630 if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED) 631 return -EINVAL; 632 633 pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table; 634 pd_entry = &pd_table->pd_entry[rel_pd_idx]; 635 if (--pd_entry->bp.use_cnt) 636 return 0; 637 638 pd_entry->valid = false; 639 pd_table->use_cnt--; 640 pd_addr = pd_table->pd_page_addr.va; 641 pd_addr += rel_pd_idx; 642 memset(pd_addr, 0, sizeof(u64)); 643 irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx); 644 645 if (!pd_entry->rsrc_pg) { 646 mem = &pd_entry->bp.addr; 647 if (!mem || !mem->va) 648 return -EINVAL; 649 650 dma_free_coherent(dev->hw->device, mem->size, mem->va, 651 mem->pa); 652 mem->va = NULL; 653 } 654 if (!pd_table->use_cnt) 655 kfree(pd_table->pd_entry_virt_mem.va); 656 657 return 0; 658 } 659 660 /** 661 * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry 662 * @hmc_info: pointer to the HMC configuration information structure 663 * @idx: the page index 664 */ 665 int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx) 666 { 667 struct irdma_hmc_sd_entry *sd_entry; 668 669 sd_entry = &hmc_info->sd_table.sd_entry[idx]; 670 if (--sd_entry->u.bp.use_cnt) 671 return -EBUSY; 672 673 hmc_info->sd_table.use_cnt--; 674 sd_entry->valid = false; 675 676 return 0; 677 } 678 679 /** 680 * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry. 681 * @hmc_info: pointer to the HMC configuration information structure 682 * @idx: segment descriptor index to find the relevant page descriptor 683 */ 684 int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx) 685 { 686 struct irdma_hmc_sd_entry *sd_entry; 687 688 sd_entry = &hmc_info->sd_table.sd_entry[idx]; 689 690 if (sd_entry->u.pd_table.use_cnt) 691 return -EBUSY; 692 693 sd_entry->valid = false; 694 hmc_info->sd_table.use_cnt--; 695 696 return 0; 697 } 698