1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include "hns_roce_device.h" 36 #include "hns_roce_hem.h" 37 #include "hns_roce_common.h" 38 39 #define DMA_ADDR_T_SHIFT 12 40 #define BT_BA_SHIFT 32 41 42 #define HEM_INDEX_BUF BIT(0) 43 #define HEM_INDEX_L0 BIT(1) 44 #define HEM_INDEX_L1 BIT(2) 45 struct hns_roce_hem_index { 46 u64 buf; 47 u64 l0; 48 u64 l1; 49 u32 inited; /* indicate which index is available */ 50 }; 51 52 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 53 { 54 int hop_num = 0; 55 56 switch (type) { 57 case HEM_TYPE_QPC: 58 hop_num = hr_dev->caps.qpc_hop_num; 59 break; 60 case HEM_TYPE_MTPT: 61 hop_num = hr_dev->caps.mpt_hop_num; 62 break; 63 case HEM_TYPE_CQC: 64 hop_num = hr_dev->caps.cqc_hop_num; 65 break; 66 case HEM_TYPE_SRQC: 67 hop_num = hr_dev->caps.srqc_hop_num; 68 break; 69 case HEM_TYPE_SCCC: 70 hop_num = hr_dev->caps.sccc_hop_num; 71 break; 72 case HEM_TYPE_QPC_TIMER: 73 hop_num = hr_dev->caps.qpc_timer_hop_num; 74 break; 75 case HEM_TYPE_CQC_TIMER: 76 hop_num = hr_dev->caps.cqc_timer_hop_num; 77 break; 78 case HEM_TYPE_CQE: 79 hop_num = hr_dev->caps.cqe_hop_num; 80 break; 81 case HEM_TYPE_MTT: 82 hop_num = hr_dev->caps.mtt_hop_num; 83 break; 84 case HEM_TYPE_SRQWQE: 85 hop_num = hr_dev->caps.srqwqe_hop_num; 86 break; 87 case HEM_TYPE_IDX: 88 hop_num = hr_dev->caps.idx_hop_num; 89 break; 90 default: 91 return false; 92 } 93 94 return hop_num ? true : false; 95 } 96 97 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx, 98 u32 bt_chunk_num, u64 hem_max_num) 99 { 100 u64 start_idx = round_down(hem_idx, bt_chunk_num); 101 u64 check_max_num = start_idx + bt_chunk_num; 102 u64 i; 103 104 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++) 105 if (i != hem_idx && hem[i]) 106 return false; 107 108 return true; 109 } 110 111 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num) 112 { 113 u64 start_idx = round_down(ba_idx, bt_chunk_num); 114 int i; 115 116 for (i = 0; i < bt_chunk_num; i++) 117 if (i != ba_idx && bt[start_idx + i]) 118 return false; 119 120 return true; 121 } 122 123 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 124 { 125 if (check_whether_bt_num_3(table_type, hop_num)) 126 return 3; 127 else if (check_whether_bt_num_2(table_type, hop_num)) 128 return 2; 129 else if (check_whether_bt_num_1(table_type, hop_num)) 130 return 1; 131 else 132 return 0; 133 } 134 135 static int get_hem_table_config(struct hns_roce_dev *hr_dev, 136 struct hns_roce_hem_mhop *mhop, 137 u32 type) 138 { 139 struct device *dev = hr_dev->dev; 140 141 switch (type) { 142 case HEM_TYPE_QPC: 143 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 144 + PAGE_SHIFT); 145 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 146 + PAGE_SHIFT); 147 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 148 mhop->hop_num = hr_dev->caps.qpc_hop_num; 149 break; 150 case HEM_TYPE_MTPT: 151 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 152 + PAGE_SHIFT); 153 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 154 + PAGE_SHIFT); 155 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 156 mhop->hop_num = hr_dev->caps.mpt_hop_num; 157 break; 158 case HEM_TYPE_CQC: 159 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 160 + PAGE_SHIFT); 161 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 162 + PAGE_SHIFT); 163 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 164 mhop->hop_num = hr_dev->caps.cqc_hop_num; 165 break; 166 case HEM_TYPE_SCCC: 167 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz 168 + PAGE_SHIFT); 169 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz 170 + PAGE_SHIFT); 171 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num; 172 mhop->hop_num = hr_dev->caps.sccc_hop_num; 173 break; 174 case HEM_TYPE_QPC_TIMER: 175 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz 176 + PAGE_SHIFT); 177 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz 178 + PAGE_SHIFT); 179 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num; 180 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num; 181 break; 182 case HEM_TYPE_CQC_TIMER: 183 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz 184 + PAGE_SHIFT); 185 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz 186 + PAGE_SHIFT); 187 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num; 188 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num; 189 break; 190 case HEM_TYPE_SRQC: 191 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 192 + PAGE_SHIFT); 193 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 194 + PAGE_SHIFT); 195 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 196 mhop->hop_num = hr_dev->caps.srqc_hop_num; 197 break; 198 case HEM_TYPE_MTT: 199 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz 200 + PAGE_SHIFT); 201 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 202 + PAGE_SHIFT); 203 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 204 mhop->hop_num = hr_dev->caps.mtt_hop_num; 205 break; 206 case HEM_TYPE_CQE: 207 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz 208 + PAGE_SHIFT); 209 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 210 + PAGE_SHIFT); 211 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 212 mhop->hop_num = hr_dev->caps.cqe_hop_num; 213 break; 214 case HEM_TYPE_SRQWQE: 215 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz 216 + PAGE_SHIFT); 217 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz 218 + PAGE_SHIFT); 219 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 220 mhop->hop_num = hr_dev->caps.srqwqe_hop_num; 221 break; 222 case HEM_TYPE_IDX: 223 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz 224 + PAGE_SHIFT); 225 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz 226 + PAGE_SHIFT); 227 mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; 228 mhop->hop_num = hr_dev->caps.idx_hop_num; 229 break; 230 default: 231 dev_err(dev, "Table %d not support multi-hop addressing!\n", 232 type); 233 return -EINVAL; 234 } 235 236 return 0; 237 } 238 239 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 240 struct hns_roce_hem_table *table, unsigned long *obj, 241 struct hns_roce_hem_mhop *mhop) 242 { 243 struct device *dev = hr_dev->dev; 244 u32 chunk_ba_num; 245 u32 table_idx; 246 u32 bt_num; 247 u32 chunk_size; 248 249 if (get_hem_table_config(hr_dev, mhop, table->type)) 250 return -EINVAL; 251 252 if (!obj) 253 return 0; 254 255 /* 256 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages. 257 * MTT/CQE alloc hem for bt pages. 258 */ 259 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 260 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 261 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 262 mhop->bt_chunk_size; 263 table_idx = (*obj & (table->num_obj - 1)) / 264 (chunk_size / table->obj_size); 265 switch (bt_num) { 266 case 3: 267 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 268 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 269 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num; 270 break; 271 case 2: 272 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 273 mhop->l0_idx = table_idx / chunk_ba_num; 274 break; 275 case 1: 276 mhop->l0_idx = table_idx; 277 break; 278 default: 279 dev_err(dev, "Table %d not support hop_num = %d!\n", 280 table->type, mhop->hop_num); 281 return -EINVAL; 282 } 283 if (mhop->l0_idx >= mhop->ba_l0_num) 284 mhop->l0_idx %= mhop->ba_l0_num; 285 286 return 0; 287 } 288 289 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 290 int npages, 291 unsigned long hem_alloc_size, 292 gfp_t gfp_mask) 293 { 294 struct hns_roce_hem_chunk *chunk = NULL; 295 struct hns_roce_hem *hem; 296 struct scatterlist *mem; 297 int order; 298 void *buf; 299 300 WARN_ON(gfp_mask & __GFP_HIGHMEM); 301 302 hem = kmalloc(sizeof(*hem), 303 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 304 if (!hem) 305 return NULL; 306 307 hem->refcount = 0; 308 INIT_LIST_HEAD(&hem->chunk_list); 309 310 order = get_order(hem_alloc_size); 311 312 while (npages > 0) { 313 if (!chunk) { 314 chunk = kmalloc(sizeof(*chunk), 315 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 316 if (!chunk) 317 goto fail; 318 319 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 320 chunk->npages = 0; 321 chunk->nsg = 0; 322 memset(chunk->buf, 0, sizeof(chunk->buf)); 323 list_add_tail(&chunk->list, &hem->chunk_list); 324 } 325 326 while (1 << order > npages) 327 --order; 328 329 /* 330 * Alloc memory one time. If failed, don't alloc small block 331 * memory, directly return fail. 332 */ 333 mem = &chunk->mem[chunk->npages]; 334 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 335 &sg_dma_address(mem), gfp_mask); 336 if (!buf) 337 goto fail; 338 339 chunk->buf[chunk->npages] = buf; 340 sg_dma_len(mem) = PAGE_SIZE << order; 341 342 ++chunk->npages; 343 ++chunk->nsg; 344 npages -= 1 << order; 345 } 346 347 return hem; 348 349 fail: 350 hns_roce_free_hem(hr_dev, hem); 351 return NULL; 352 } 353 354 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 355 { 356 struct hns_roce_hem_chunk *chunk, *tmp; 357 int i; 358 359 if (!hem) 360 return; 361 362 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 363 for (i = 0; i < chunk->npages; ++i) 364 dma_free_coherent(hr_dev->dev, 365 sg_dma_len(&chunk->mem[i]), 366 chunk->buf[i], 367 sg_dma_address(&chunk->mem[i])); 368 kfree(chunk); 369 } 370 371 kfree(hem); 372 } 373 374 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, 375 struct hns_roce_hem_table *table, unsigned long obj) 376 { 377 spinlock_t *lock = &hr_dev->bt_cmd_lock; 378 struct device *dev = hr_dev->dev; 379 long end; 380 unsigned long flags; 381 struct hns_roce_hem_iter iter; 382 void __iomem *bt_cmd; 383 __le32 bt_cmd_val[2]; 384 __le32 bt_cmd_h = 0; 385 __le32 bt_cmd_l = 0; 386 u64 bt_ba = 0; 387 int ret = 0; 388 389 /* Find the HEM(Hardware Entry Memory) entry */ 390 unsigned long i = (obj & (table->num_obj - 1)) / 391 (table->table_chunk_size / table->obj_size); 392 393 switch (table->type) { 394 case HEM_TYPE_QPC: 395 case HEM_TYPE_MTPT: 396 case HEM_TYPE_CQC: 397 case HEM_TYPE_SRQC: 398 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 399 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type); 400 break; 401 default: 402 return ret; 403 } 404 405 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, 406 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); 407 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); 408 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); 409 410 /* Currently iter only a chunk */ 411 for (hns_roce_hem_first(table->hem[i], &iter); 412 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 413 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; 414 415 spin_lock_irqsave(lock, flags); 416 417 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; 418 419 end = HW_SYNC_TIMEOUT_MSECS; 420 while (end > 0) { 421 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) 422 break; 423 424 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); 425 end -= HW_SYNC_SLEEP_TIME_INTERVAL; 426 } 427 428 if (end <= 0) { 429 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); 430 spin_unlock_irqrestore(lock, flags); 431 return -EBUSY; 432 } 433 434 bt_cmd_l = cpu_to_le32(bt_ba); 435 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, 436 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 437 bt_ba >> BT_BA_SHIFT); 438 439 bt_cmd_val[0] = bt_cmd_l; 440 bt_cmd_val[1] = bt_cmd_h; 441 hns_roce_write64_k(bt_cmd_val, 442 hr_dev->reg_base + ROCEE_BT_CMD_L_REG); 443 spin_unlock_irqrestore(lock, flags); 444 } 445 446 return ret; 447 } 448 449 static int calc_hem_config(struct hns_roce_dev *hr_dev, 450 struct hns_roce_hem_table *table, unsigned long obj, 451 struct hns_roce_hem_mhop *mhop, 452 struct hns_roce_hem_index *index) 453 { 454 struct ib_device *ibdev = &hr_dev->ib_dev; 455 unsigned long mhop_obj = obj; 456 u32 l0_idx, l1_idx, l2_idx; 457 u32 chunk_ba_num; 458 u32 bt_num; 459 int ret; 460 461 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop); 462 if (ret) 463 return ret; 464 465 l0_idx = mhop->l0_idx; 466 l1_idx = mhop->l1_idx; 467 l2_idx = mhop->l2_idx; 468 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 469 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 470 switch (bt_num) { 471 case 3: 472 index->l1 = l0_idx * chunk_ba_num + l1_idx; 473 index->l0 = l0_idx; 474 index->buf = l0_idx * chunk_ba_num * chunk_ba_num + 475 l1_idx * chunk_ba_num + l2_idx; 476 break; 477 case 2: 478 index->l0 = l0_idx; 479 index->buf = l0_idx * chunk_ba_num + l1_idx; 480 break; 481 case 1: 482 index->buf = l0_idx; 483 break; 484 default: 485 ibdev_err(ibdev, "Table %d not support mhop.hop_num = %d!\n", 486 table->type, mhop->hop_num); 487 return -EINVAL; 488 } 489 490 if (unlikely(index->buf >= table->num_hem)) { 491 ibdev_err(ibdev, "Table %d exceed hem limt idx %llu,max %lu!\n", 492 table->type, index->buf, table->num_hem); 493 return -EINVAL; 494 } 495 496 return 0; 497 } 498 499 static void free_mhop_hem(struct hns_roce_dev *hr_dev, 500 struct hns_roce_hem_table *table, 501 struct hns_roce_hem_mhop *mhop, 502 struct hns_roce_hem_index *index) 503 { 504 u32 bt_size = mhop->bt_chunk_size; 505 struct device *dev = hr_dev->dev; 506 507 if (index->inited & HEM_INDEX_BUF) { 508 hns_roce_free_hem(hr_dev, table->hem[index->buf]); 509 table->hem[index->buf] = NULL; 510 } 511 512 if (index->inited & HEM_INDEX_L1) { 513 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1], 514 table->bt_l1_dma_addr[index->l1]); 515 table->bt_l1[index->l1] = NULL; 516 } 517 518 if (index->inited & HEM_INDEX_L0) { 519 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0], 520 table->bt_l0_dma_addr[index->l0]); 521 table->bt_l0[index->l0] = NULL; 522 } 523 } 524 525 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev, 526 struct hns_roce_hem_table *table, 527 struct hns_roce_hem_mhop *mhop, 528 struct hns_roce_hem_index *index) 529 { 530 u32 bt_size = mhop->bt_chunk_size; 531 struct device *dev = hr_dev->dev; 532 struct hns_roce_hem_iter iter; 533 gfp_t flag; 534 u64 bt_ba; 535 u32 size; 536 int ret; 537 538 /* alloc L1 BA's chunk */ 539 if ((check_whether_bt_num_3(table->type, mhop->hop_num) || 540 check_whether_bt_num_2(table->type, mhop->hop_num)) && 541 !table->bt_l0[index->l0]) { 542 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size, 543 &table->bt_l0_dma_addr[index->l0], 544 GFP_KERNEL); 545 if (!table->bt_l0[index->l0]) { 546 ret = -ENOMEM; 547 goto out; 548 } 549 index->inited |= HEM_INDEX_L0; 550 } 551 552 /* alloc L2 BA's chunk */ 553 if (check_whether_bt_num_3(table->type, mhop->hop_num) && 554 !table->bt_l1[index->l1]) { 555 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size, 556 &table->bt_l1_dma_addr[index->l1], 557 GFP_KERNEL); 558 if (!table->bt_l1[index->l1]) { 559 ret = -ENOMEM; 560 goto err_alloc_hem; 561 } 562 index->inited |= HEM_INDEX_L1; 563 *(table->bt_l0[index->l0] + mhop->l1_idx) = 564 table->bt_l1_dma_addr[index->l1]; 565 } 566 567 /* 568 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC. 569 * alloc bt space chunk for MTT/CQE. 570 */ 571 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size; 572 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN; 573 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT, 574 size, flag); 575 if (!table->hem[index->buf]) { 576 ret = -ENOMEM; 577 goto err_alloc_hem; 578 } 579 580 index->inited |= HEM_INDEX_BUF; 581 hns_roce_hem_first(table->hem[index->buf], &iter); 582 bt_ba = hns_roce_hem_addr(&iter); 583 if (table->type < HEM_TYPE_MTT) { 584 if (mhop->hop_num == 2) 585 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba; 586 else if (mhop->hop_num == 1) 587 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 588 } else if (mhop->hop_num == 2) { 589 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba; 590 } 591 592 return 0; 593 err_alloc_hem: 594 free_mhop_hem(hr_dev, table, mhop, index); 595 out: 596 return ret; 597 } 598 599 static int set_mhop_hem(struct hns_roce_dev *hr_dev, 600 struct hns_roce_hem_table *table, unsigned long obj, 601 struct hns_roce_hem_mhop *mhop, 602 struct hns_roce_hem_index *index) 603 { 604 struct ib_device *ibdev = &hr_dev->ib_dev; 605 int step_idx; 606 int ret = 0; 607 608 if (index->inited & HEM_INDEX_L0) { 609 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0); 610 if (ret) { 611 ibdev_err(ibdev, "set HEM step 0 failed!\n"); 612 goto out; 613 } 614 } 615 616 if (index->inited & HEM_INDEX_L1) { 617 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1); 618 if (ret) { 619 ibdev_err(ibdev, "set HEM step 1 failed!\n"); 620 goto out; 621 } 622 } 623 624 if (index->inited & HEM_INDEX_BUF) { 625 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0) 626 step_idx = 0; 627 else 628 step_idx = mhop->hop_num; 629 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx); 630 if (ret) 631 ibdev_err(ibdev, "set HEM step last failed!\n"); 632 } 633 out: 634 return ret; 635 } 636 637 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 638 struct hns_roce_hem_table *table, 639 unsigned long obj) 640 { 641 struct ib_device *ibdev = &hr_dev->ib_dev; 642 struct hns_roce_hem_index index = {}; 643 struct hns_roce_hem_mhop mhop = {}; 644 int ret; 645 646 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 647 if (ret) { 648 ibdev_err(ibdev, "calc hem config failed!\n"); 649 return ret; 650 } 651 652 mutex_lock(&table->mutex); 653 if (table->hem[index.buf]) { 654 ++table->hem[index.buf]->refcount; 655 goto out; 656 } 657 658 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index); 659 if (ret) { 660 ibdev_err(ibdev, "alloc mhop hem failed!\n"); 661 goto out; 662 } 663 664 /* set HEM base address to hardware */ 665 if (table->type < HEM_TYPE_MTT) { 666 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index); 667 if (ret) { 668 ibdev_err(ibdev, "set HEM address to HW failed!\n"); 669 goto err_alloc; 670 } 671 } 672 673 ++table->hem[index.buf]->refcount; 674 goto out; 675 676 err_alloc: 677 free_mhop_hem(hr_dev, table, &mhop, &index); 678 out: 679 mutex_unlock(&table->mutex); 680 return ret; 681 } 682 683 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 684 struct hns_roce_hem_table *table, unsigned long obj) 685 { 686 struct device *dev = hr_dev->dev; 687 int ret = 0; 688 unsigned long i; 689 690 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 691 return hns_roce_table_mhop_get(hr_dev, table, obj); 692 693 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / 694 table->obj_size); 695 696 mutex_lock(&table->mutex); 697 698 if (table->hem[i]) { 699 ++table->hem[i]->refcount; 700 goto out; 701 } 702 703 table->hem[i] = hns_roce_alloc_hem(hr_dev, 704 table->table_chunk_size >> PAGE_SHIFT, 705 table->table_chunk_size, 706 (table->lowmem ? GFP_KERNEL : 707 GFP_HIGHUSER) | __GFP_NOWARN); 708 if (!table->hem[i]) { 709 ret = -ENOMEM; 710 goto out; 711 } 712 713 /* Set HEM base address(128K/page, pa) to Hardware */ 714 if (hns_roce_set_hem(hr_dev, table, obj)) { 715 hns_roce_free_hem(hr_dev, table->hem[i]); 716 table->hem[i] = NULL; 717 ret = -ENODEV; 718 dev_err(dev, "set HEM base address to HW failed.\n"); 719 goto out; 720 } 721 722 ++table->hem[i]->refcount; 723 out: 724 mutex_unlock(&table->mutex); 725 return ret; 726 } 727 728 static void clear_mhop_hem(struct hns_roce_dev *hr_dev, 729 struct hns_roce_hem_table *table, unsigned long obj, 730 struct hns_roce_hem_mhop *mhop, 731 struct hns_roce_hem_index *index) 732 { 733 struct ib_device *ibdev = &hr_dev->ib_dev; 734 u32 hop_num = mhop->hop_num; 735 u32 chunk_ba_num; 736 int step_idx; 737 738 index->inited = HEM_INDEX_BUF; 739 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; 740 if (check_whether_bt_num_2(table->type, hop_num)) { 741 if (hns_roce_check_hem_null(table->hem, index->buf, 742 chunk_ba_num, table->num_hem)) 743 index->inited |= HEM_INDEX_L0; 744 } else if (check_whether_bt_num_3(table->type, hop_num)) { 745 if (hns_roce_check_hem_null(table->hem, index->buf, 746 chunk_ba_num, table->num_hem)) { 747 index->inited |= HEM_INDEX_L1; 748 if (hns_roce_check_bt_null(table->bt_l1, index->l1, 749 chunk_ba_num)) 750 index->inited |= HEM_INDEX_L0; 751 } 752 } 753 754 if (table->type < HEM_TYPE_MTT) { 755 if (hop_num == HNS_ROCE_HOP_NUM_0) 756 step_idx = 0; 757 else 758 step_idx = hop_num; 759 760 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx)) 761 ibdev_warn(ibdev, "Clear hop%d HEM failed.\n", hop_num); 762 763 if (index->inited & HEM_INDEX_L1) 764 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 765 ibdev_warn(ibdev, "Clear HEM step 1 failed.\n"); 766 767 if (index->inited & HEM_INDEX_L0) 768 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 769 ibdev_warn(ibdev, "Clear HEM step 0 failed.\n"); 770 } 771 } 772 773 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 774 struct hns_roce_hem_table *table, 775 unsigned long obj, 776 int check_refcount) 777 { 778 struct ib_device *ibdev = &hr_dev->ib_dev; 779 struct hns_roce_hem_index index = {}; 780 struct hns_roce_hem_mhop mhop = {}; 781 int ret; 782 783 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index); 784 if (ret) { 785 ibdev_err(ibdev, "calc hem config failed!\n"); 786 return; 787 } 788 789 mutex_lock(&table->mutex); 790 if (check_refcount && (--table->hem[index.buf]->refcount > 0)) { 791 mutex_unlock(&table->mutex); 792 return; 793 } 794 795 clear_mhop_hem(hr_dev, table, obj, &mhop, &index); 796 free_mhop_hem(hr_dev, table, &mhop, &index); 797 798 mutex_unlock(&table->mutex); 799 } 800 801 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 802 struct hns_roce_hem_table *table, unsigned long obj) 803 { 804 struct device *dev = hr_dev->dev; 805 unsigned long i; 806 807 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 808 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 809 return; 810 } 811 812 i = (obj & (table->num_obj - 1)) / 813 (table->table_chunk_size / table->obj_size); 814 815 mutex_lock(&table->mutex); 816 817 if (--table->hem[i]->refcount == 0) { 818 /* Clear HEM base address */ 819 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 820 dev_warn(dev, "Clear HEM base address failed.\n"); 821 822 hns_roce_free_hem(hr_dev, table->hem[i]); 823 table->hem[i] = NULL; 824 } 825 826 mutex_unlock(&table->mutex); 827 } 828 829 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 830 struct hns_roce_hem_table *table, 831 unsigned long obj, dma_addr_t *dma_handle) 832 { 833 struct hns_roce_hem_chunk *chunk; 834 struct hns_roce_hem_mhop mhop; 835 struct hns_roce_hem *hem; 836 void *addr = NULL; 837 unsigned long mhop_obj = obj; 838 unsigned long obj_per_chunk; 839 unsigned long idx_offset; 840 int offset, dma_offset; 841 int length; 842 int i, j; 843 u32 hem_idx = 0; 844 845 if (!table->lowmem) 846 return NULL; 847 848 mutex_lock(&table->mutex); 849 850 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 851 obj_per_chunk = table->table_chunk_size / table->obj_size; 852 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; 853 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; 854 dma_offset = offset = idx_offset * table->obj_size; 855 } else { 856 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */ 857 858 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop)) 859 goto out; 860 /* mtt mhop */ 861 i = mhop.l0_idx; 862 j = mhop.l1_idx; 863 if (mhop.hop_num == 2) 864 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; 865 else if (mhop.hop_num == 1 || 866 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 867 hem_idx = i; 868 869 hem = table->hem[hem_idx]; 870 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size % 871 mhop.bt_chunk_size; 872 if (mhop.hop_num == 2) 873 dma_offset = offset = 0; 874 } 875 876 if (!hem) 877 goto out; 878 879 list_for_each_entry(chunk, &hem->chunk_list, list) { 880 for (i = 0; i < chunk->npages; ++i) { 881 length = sg_dma_len(&chunk->mem[i]); 882 if (dma_handle && dma_offset >= 0) { 883 if (length > (u32)dma_offset) 884 *dma_handle = sg_dma_address( 885 &chunk->mem[i]) + dma_offset; 886 dma_offset -= length; 887 } 888 889 if (length > (u32)offset) { 890 addr = chunk->buf[i] + offset; 891 goto out; 892 } 893 offset -= length; 894 } 895 } 896 897 out: 898 mutex_unlock(&table->mutex); 899 return addr; 900 } 901 902 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, 903 struct hns_roce_hem_table *table, 904 unsigned long start, unsigned long end) 905 { 906 struct hns_roce_hem_mhop mhop; 907 unsigned long inc = table->table_chunk_size / table->obj_size; 908 unsigned long i = 0; 909 int ret; 910 911 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 912 ret = hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 913 if (ret) 914 goto fail; 915 inc = mhop.bt_chunk_size / table->obj_size; 916 } 917 918 /* Allocate MTT entry memory according to chunk(128K) */ 919 for (i = start; i <= end; i += inc) { 920 ret = hns_roce_table_get(hr_dev, table, i); 921 if (ret) 922 goto fail; 923 } 924 925 return 0; 926 927 fail: 928 while (i > start) { 929 i -= inc; 930 hns_roce_table_put(hr_dev, table, i); 931 } 932 return ret; 933 } 934 935 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, 936 struct hns_roce_hem_table *table, 937 unsigned long start, unsigned long end) 938 { 939 struct hns_roce_hem_mhop mhop; 940 unsigned long inc = table->table_chunk_size / table->obj_size; 941 unsigned long i; 942 943 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 944 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 945 return; 946 inc = mhop.bt_chunk_size / table->obj_size; 947 } 948 949 for (i = start; i <= end; i += inc) 950 hns_roce_table_put(hr_dev, table, i); 951 } 952 953 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 954 struct hns_roce_hem_table *table, u32 type, 955 unsigned long obj_size, unsigned long nobj, 956 int use_lowmem) 957 { 958 unsigned long obj_per_chunk; 959 unsigned long num_hem; 960 961 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 962 table->table_chunk_size = hr_dev->caps.chunk_sz; 963 obj_per_chunk = table->table_chunk_size / obj_size; 964 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 965 966 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 967 if (!table->hem) 968 return -ENOMEM; 969 } else { 970 struct hns_roce_hem_mhop mhop = {}; 971 unsigned long buf_chunk_size; 972 unsigned long bt_chunk_size; 973 unsigned long bt_chunk_num; 974 unsigned long num_bt_l0 = 0; 975 u32 hop_num; 976 977 if (get_hem_table_config(hr_dev, &mhop, type)) 978 return -EINVAL; 979 980 buf_chunk_size = mhop.buf_chunk_size; 981 bt_chunk_size = mhop.bt_chunk_size; 982 num_bt_l0 = mhop.ba_l0_num; 983 hop_num = mhop.hop_num; 984 985 obj_per_chunk = buf_chunk_size / obj_size; 986 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 987 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; 988 if (type >= HEM_TYPE_MTT) 989 num_bt_l0 = bt_chunk_num; 990 991 table->hem = kcalloc(num_hem, sizeof(*table->hem), 992 GFP_KERNEL); 993 if (!table->hem) 994 goto err_kcalloc_hem_buf; 995 996 if (check_whether_bt_num_3(type, hop_num)) { 997 unsigned long num_bt_l1; 998 999 num_bt_l1 = (num_hem + bt_chunk_num - 1) / 1000 bt_chunk_num; 1001 table->bt_l1 = kcalloc(num_bt_l1, 1002 sizeof(*table->bt_l1), 1003 GFP_KERNEL); 1004 if (!table->bt_l1) 1005 goto err_kcalloc_bt_l1; 1006 1007 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 1008 sizeof(*table->bt_l1_dma_addr), 1009 GFP_KERNEL); 1010 1011 if (!table->bt_l1_dma_addr) 1012 goto err_kcalloc_l1_dma; 1013 } 1014 1015 if (check_whether_bt_num_2(type, hop_num) || 1016 check_whether_bt_num_3(type, hop_num)) { 1017 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 1018 GFP_KERNEL); 1019 if (!table->bt_l0) 1020 goto err_kcalloc_bt_l0; 1021 1022 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 1023 sizeof(*table->bt_l0_dma_addr), 1024 GFP_KERNEL); 1025 if (!table->bt_l0_dma_addr) 1026 goto err_kcalloc_l0_dma; 1027 } 1028 } 1029 1030 table->type = type; 1031 table->num_hem = num_hem; 1032 table->num_obj = nobj; 1033 table->obj_size = obj_size; 1034 table->lowmem = use_lowmem; 1035 mutex_init(&table->mutex); 1036 1037 return 0; 1038 1039 err_kcalloc_l0_dma: 1040 kfree(table->bt_l0); 1041 table->bt_l0 = NULL; 1042 1043 err_kcalloc_bt_l0: 1044 kfree(table->bt_l1_dma_addr); 1045 table->bt_l1_dma_addr = NULL; 1046 1047 err_kcalloc_l1_dma: 1048 kfree(table->bt_l1); 1049 table->bt_l1 = NULL; 1050 1051 err_kcalloc_bt_l1: 1052 kfree(table->hem); 1053 table->hem = NULL; 1054 1055 err_kcalloc_hem_buf: 1056 return -ENOMEM; 1057 } 1058 1059 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 1060 struct hns_roce_hem_table *table) 1061 { 1062 struct hns_roce_hem_mhop mhop; 1063 u32 buf_chunk_size; 1064 int i; 1065 u64 obj; 1066 1067 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop)) 1068 return; 1069 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 1070 mhop.bt_chunk_size; 1071 1072 for (i = 0; i < table->num_hem; ++i) { 1073 obj = i * buf_chunk_size / table->obj_size; 1074 if (table->hem[i]) 1075 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 1076 } 1077 1078 kfree(table->hem); 1079 table->hem = NULL; 1080 kfree(table->bt_l1); 1081 table->bt_l1 = NULL; 1082 kfree(table->bt_l1_dma_addr); 1083 table->bt_l1_dma_addr = NULL; 1084 kfree(table->bt_l0); 1085 table->bt_l0 = NULL; 1086 kfree(table->bt_l0_dma_addr); 1087 table->bt_l0_dma_addr = NULL; 1088 } 1089 1090 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 1091 struct hns_roce_hem_table *table) 1092 { 1093 struct device *dev = hr_dev->dev; 1094 unsigned long i; 1095 1096 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 1097 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 1098 return; 1099 } 1100 1101 for (i = 0; i < table->num_hem; ++i) 1102 if (table->hem[i]) { 1103 if (hr_dev->hw->clear_hem(hr_dev, table, 1104 i * table->table_chunk_size / table->obj_size, 0)) 1105 dev_err(dev, "Clear HEM base address failed.\n"); 1106 1107 hns_roce_free_hem(hr_dev, table->hem[i]); 1108 } 1109 1110 kfree(table->hem); 1111 } 1112 1113 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 1114 { 1115 if ((hr_dev->caps.num_idx_segs)) 1116 hns_roce_cleanup_hem_table(hr_dev, 1117 &hr_dev->mr_table.mtt_idx_table); 1118 if (hr_dev->caps.num_srqwqe_segs) 1119 hns_roce_cleanup_hem_table(hr_dev, 1120 &hr_dev->mr_table.mtt_srqwqe_table); 1121 if (hr_dev->caps.srqc_entry_sz) 1122 hns_roce_cleanup_hem_table(hr_dev, 1123 &hr_dev->srq_table.table); 1124 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 1125 if (hr_dev->caps.qpc_timer_entry_sz) 1126 hns_roce_cleanup_hem_table(hr_dev, 1127 &hr_dev->qpc_timer_table); 1128 if (hr_dev->caps.cqc_timer_entry_sz) 1129 hns_roce_cleanup_hem_table(hr_dev, 1130 &hr_dev->cqc_timer_table); 1131 if (hr_dev->caps.sccc_entry_sz) 1132 hns_roce_cleanup_hem_table(hr_dev, 1133 &hr_dev->qp_table.sccc_table); 1134 if (hr_dev->caps.trrl_entry_sz) 1135 hns_roce_cleanup_hem_table(hr_dev, 1136 &hr_dev->qp_table.trrl_table); 1137 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 1138 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 1139 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1140 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 1141 hns_roce_cleanup_hem_table(hr_dev, 1142 &hr_dev->mr_table.mtt_cqe_table); 1143 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); 1144 } 1145 1146 struct roce_hem_item { 1147 struct list_head list; /* link all hems in the same bt level */ 1148 struct list_head sibling; /* link all hems in last hop for mtt */ 1149 void *addr; 1150 dma_addr_t dma_addr; 1151 size_t count; /* max ba numbers */ 1152 int start; /* start buf offset in this hem */ 1153 int end; /* end buf offset in this hem */ 1154 }; 1155 1156 static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, 1157 int start, int end, 1158 int count, bool exist_bt, 1159 int bt_level) 1160 { 1161 struct roce_hem_item *hem; 1162 1163 hem = kzalloc(sizeof(*hem), GFP_KERNEL); 1164 if (!hem) 1165 return NULL; 1166 1167 if (exist_bt) { 1168 hem->addr = dma_alloc_coherent(hr_dev->dev, 1169 count * BA_BYTE_LEN, 1170 &hem->dma_addr, GFP_KERNEL); 1171 if (!hem->addr) { 1172 kfree(hem); 1173 return NULL; 1174 } 1175 } 1176 1177 hem->count = count; 1178 hem->start = start; 1179 hem->end = end; 1180 INIT_LIST_HEAD(&hem->list); 1181 INIT_LIST_HEAD(&hem->sibling); 1182 1183 return hem; 1184 } 1185 1186 static void hem_list_free_item(struct hns_roce_dev *hr_dev, 1187 struct roce_hem_item *hem, bool exist_bt) 1188 { 1189 if (exist_bt) 1190 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, 1191 hem->addr, hem->dma_addr); 1192 kfree(hem); 1193 } 1194 1195 static void hem_list_free_all(struct hns_roce_dev *hr_dev, 1196 struct list_head *head, bool exist_bt) 1197 { 1198 struct roce_hem_item *hem, *temp_hem; 1199 1200 list_for_each_entry_safe(hem, temp_hem, head, list) { 1201 list_del(&hem->list); 1202 hem_list_free_item(hr_dev, hem, exist_bt); 1203 } 1204 } 1205 1206 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, 1207 u64 table_addr) 1208 { 1209 *(u64 *)(base_addr) = table_addr; 1210 } 1211 1212 /* assign L0 table address to hem from root bt */ 1213 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, 1214 struct roce_hem_item *hem, void *cpu_addr, 1215 u64 phy_addr) 1216 { 1217 hem->addr = cpu_addr; 1218 hem->dma_addr = (dma_addr_t)phy_addr; 1219 } 1220 1221 static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, 1222 int offset) 1223 { 1224 return (hem->start <= offset && offset <= hem->end); 1225 } 1226 1227 static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, 1228 int page_offset) 1229 { 1230 struct roce_hem_item *hem, *temp_hem; 1231 struct roce_hem_item *found = NULL; 1232 1233 list_for_each_entry_safe(hem, temp_hem, ba_list, list) { 1234 if (hem_list_page_is_in_range(hem, page_offset)) { 1235 found = hem; 1236 break; 1237 } 1238 } 1239 1240 return found; 1241 } 1242 1243 static bool hem_list_is_bottom_bt(int hopnum, int bt_level) 1244 { 1245 /* 1246 * hopnum base address table levels 1247 * 0 L0(buf) 1248 * 1 L0 -> buf 1249 * 2 L0 -> L1 -> buf 1250 * 3 L0 -> L1 -> L2 -> buf 1251 */ 1252 return bt_level >= (hopnum ? hopnum - 1 : hopnum); 1253 } 1254 1255 /** 1256 * calc base address entries num 1257 * @hopnum: num of mutihop addressing 1258 * @bt_level: base address table level 1259 * @unit: ba entries per bt page 1260 */ 1261 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) 1262 { 1263 u32 step; 1264 int max; 1265 int i; 1266 1267 if (hopnum <= bt_level) 1268 return 0; 1269 /* 1270 * hopnum bt_level range 1271 * 1 0 unit 1272 * ------------ 1273 * 2 0 unit * unit 1274 * 2 1 unit 1275 * ------------ 1276 * 3 0 unit * unit * unit 1277 * 3 1 unit * unit 1278 * 3 2 unit 1279 */ 1280 step = 1; 1281 max = hopnum - bt_level; 1282 for (i = 0; i < max; i++) 1283 step = step * unit; 1284 1285 return step; 1286 } 1287 1288 /** 1289 * calc the root ba entries which could cover all regions 1290 * @regions: buf region array 1291 * @region_cnt: array size of @regions 1292 * @unit: ba entries per bt page 1293 */ 1294 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, 1295 int region_cnt, int unit) 1296 { 1297 struct hns_roce_buf_region *r; 1298 int total = 0; 1299 int step; 1300 int i; 1301 1302 for (i = 0; i < region_cnt; i++) { 1303 r = (struct hns_roce_buf_region *)®ions[i]; 1304 if (r->hopnum > 1) { 1305 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1306 if (step > 0) 1307 total += (r->count + step - 1) / step; 1308 } else { 1309 total += r->count; 1310 } 1311 } 1312 1313 return total; 1314 } 1315 1316 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, 1317 const struct hns_roce_buf_region *r, int unit, 1318 int offset, struct list_head *mid_bt, 1319 struct list_head *btm_bt) 1320 { 1321 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; 1322 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; 1323 struct roce_hem_item *cur, *pre; 1324 const int hopnum = r->hopnum; 1325 int start_aligned; 1326 int distance; 1327 int ret = 0; 1328 int max_ofs; 1329 int level; 1330 u32 step; 1331 int end; 1332 1333 if (hopnum <= 1) 1334 return 0; 1335 1336 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { 1337 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); 1338 return -EINVAL; 1339 } 1340 1341 if (offset < r->offset) { 1342 dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", 1343 offset, r->offset); 1344 return -EINVAL; 1345 } 1346 1347 distance = offset - r->offset; 1348 max_ofs = r->offset + r->count - 1; 1349 for (level = 0; level < hopnum; level++) 1350 INIT_LIST_HEAD(&temp_list[level]); 1351 1352 /* config L1 bt to last bt and link them to corresponding parent */ 1353 for (level = 1; level < hopnum; level++) { 1354 cur = hem_list_search_item(&mid_bt[level], offset); 1355 if (cur) { 1356 hem_ptrs[level] = cur; 1357 continue; 1358 } 1359 1360 step = hem_list_calc_ba_range(hopnum, level, unit); 1361 if (step < 1) { 1362 ret = -EINVAL; 1363 goto err_exit; 1364 } 1365 1366 start_aligned = (distance / step) * step + r->offset; 1367 end = min_t(int, start_aligned + step - 1, max_ofs); 1368 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, 1369 true, level); 1370 if (!cur) { 1371 ret = -ENOMEM; 1372 goto err_exit; 1373 } 1374 hem_ptrs[level] = cur; 1375 list_add(&cur->list, &temp_list[level]); 1376 if (hem_list_is_bottom_bt(hopnum, level)) 1377 list_add(&cur->sibling, &temp_list[0]); 1378 1379 /* link bt to parent bt */ 1380 if (level > 1) { 1381 pre = hem_ptrs[level - 1]; 1382 step = (cur->start - pre->start) / step * BA_BYTE_LEN; 1383 hem_list_link_bt(hr_dev, pre->addr + step, 1384 cur->dma_addr); 1385 } 1386 } 1387 1388 list_splice(&temp_list[0], btm_bt); 1389 for (level = 1; level < hopnum; level++) 1390 list_splice(&temp_list[level], &mid_bt[level]); 1391 1392 return 0; 1393 1394 err_exit: 1395 for (level = 1; level < hopnum; level++) 1396 hem_list_free_all(hr_dev, &temp_list[level], true); 1397 1398 return ret; 1399 } 1400 1401 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, 1402 struct hns_roce_hem_list *hem_list, int unit, 1403 const struct hns_roce_buf_region *regions, 1404 int region_cnt) 1405 { 1406 struct roce_hem_item *hem, *temp_hem, *root_hem; 1407 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; 1408 const struct hns_roce_buf_region *r; 1409 struct list_head temp_root; 1410 struct list_head temp_btm; 1411 void *cpu_base; 1412 u64 phy_base; 1413 int ret = 0; 1414 int ba_num; 1415 int offset; 1416 int total; 1417 int step; 1418 int i; 1419 1420 r = ®ions[0]; 1421 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); 1422 if (root_hem) 1423 return 0; 1424 1425 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit); 1426 if (ba_num < 1) 1427 return -ENOMEM; 1428 1429 INIT_LIST_HEAD(&temp_root); 1430 offset = r->offset; 1431 /* indicate to last region */ 1432 r = ®ions[region_cnt - 1]; 1433 root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1, 1434 ba_num, true, 0); 1435 if (!root_hem) 1436 return -ENOMEM; 1437 list_add(&root_hem->list, &temp_root); 1438 1439 hem_list->root_ba = root_hem->dma_addr; 1440 1441 INIT_LIST_HEAD(&temp_btm); 1442 for (i = 0; i < region_cnt; i++) 1443 INIT_LIST_HEAD(&temp_list[i]); 1444 1445 total = 0; 1446 for (i = 0; i < region_cnt && total < ba_num; i++) { 1447 r = ®ions[i]; 1448 if (!r->count) 1449 continue; 1450 1451 /* all regions's mid[x][0] shared the root_bt's trunk */ 1452 cpu_base = root_hem->addr + total * BA_BYTE_LEN; 1453 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; 1454 1455 /* if hopnum is 0 or 1, cut a new fake hem from the root bt 1456 * which's address share to all regions. 1457 */ 1458 if (hem_list_is_bottom_bt(r->hopnum, 0)) { 1459 hem = hem_list_alloc_item(hr_dev, r->offset, 1460 r->offset + r->count - 1, 1461 r->count, false, 0); 1462 if (!hem) { 1463 ret = -ENOMEM; 1464 goto err_exit; 1465 } 1466 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); 1467 list_add(&hem->list, &temp_list[i]); 1468 list_add(&hem->sibling, &temp_btm); 1469 total += r->count; 1470 } else { 1471 step = hem_list_calc_ba_range(r->hopnum, 1, unit); 1472 if (step < 1) { 1473 ret = -EINVAL; 1474 goto err_exit; 1475 } 1476 /* if exist mid bt, link L1 to L0 */ 1477 list_for_each_entry_safe(hem, temp_hem, 1478 &hem_list->mid_bt[i][1], list) { 1479 offset = (hem->start - r->offset) / step * 1480 BA_BYTE_LEN; 1481 hem_list_link_bt(hr_dev, cpu_base + offset, 1482 hem->dma_addr); 1483 total++; 1484 } 1485 } 1486 } 1487 1488 list_splice(&temp_btm, &hem_list->btm_bt); 1489 list_splice(&temp_root, &hem_list->root_bt); 1490 for (i = 0; i < region_cnt; i++) 1491 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); 1492 1493 return 0; 1494 1495 err_exit: 1496 for (i = 0; i < region_cnt; i++) 1497 hem_list_free_all(hr_dev, &temp_list[i], false); 1498 1499 hem_list_free_all(hr_dev, &temp_root, true); 1500 1501 return ret; 1502 } 1503 1504 /* construct the base address table and link them by address hop config */ 1505 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, 1506 struct hns_roce_hem_list *hem_list, 1507 const struct hns_roce_buf_region *regions, 1508 int region_cnt) 1509 { 1510 const struct hns_roce_buf_region *r; 1511 int ofs, end; 1512 int ret = 0; 1513 int unit; 1514 int i; 1515 1516 if (region_cnt > HNS_ROCE_MAX_BT_REGION) { 1517 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", 1518 region_cnt); 1519 return -EINVAL; 1520 } 1521 1522 unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN; 1523 for (i = 0; i < region_cnt; i++) { 1524 r = ®ions[i]; 1525 if (!r->count) 1526 continue; 1527 1528 end = r->offset + r->count; 1529 for (ofs = r->offset; ofs < end; ofs += unit) { 1530 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, 1531 hem_list->mid_bt[i], 1532 &hem_list->btm_bt); 1533 if (ret) { 1534 dev_err(hr_dev->dev, 1535 "alloc hem trunk fail ret=%d!\n", ret); 1536 goto err_alloc; 1537 } 1538 } 1539 } 1540 1541 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, 1542 region_cnt); 1543 if (ret) 1544 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); 1545 else 1546 return 0; 1547 1548 err_alloc: 1549 hns_roce_hem_list_release(hr_dev, hem_list); 1550 1551 return ret; 1552 } 1553 1554 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, 1555 struct hns_roce_hem_list *hem_list) 1556 { 1557 int i, j; 1558 1559 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1560 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1561 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], 1562 j != 0); 1563 1564 hem_list_free_all(hr_dev, &hem_list->root_bt, true); 1565 INIT_LIST_HEAD(&hem_list->btm_bt); 1566 hem_list->root_ba = 0; 1567 } 1568 1569 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, 1570 int bt_page_order) 1571 { 1572 int i, j; 1573 1574 INIT_LIST_HEAD(&hem_list->root_bt); 1575 INIT_LIST_HEAD(&hem_list->btm_bt); 1576 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) 1577 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) 1578 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); 1579 1580 hem_list->bt_pg_shift = bt_page_order; 1581 } 1582 1583 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, 1584 struct hns_roce_hem_list *hem_list, 1585 int offset, int *mtt_cnt, u64 *phy_addr) 1586 { 1587 struct list_head *head = &hem_list->btm_bt; 1588 struct roce_hem_item *hem, *temp_hem; 1589 void *cpu_base = NULL; 1590 u64 phy_base = 0; 1591 int nr = 0; 1592 1593 list_for_each_entry_safe(hem, temp_hem, head, sibling) { 1594 if (hem_list_page_is_in_range(hem, offset)) { 1595 nr = offset - hem->start; 1596 cpu_base = hem->addr + nr * BA_BYTE_LEN; 1597 phy_base = hem->dma_addr + nr * BA_BYTE_LEN; 1598 nr = hem->end + 1 - offset; 1599 break; 1600 } 1601 } 1602 1603 if (mtt_cnt) 1604 *mtt_cnt = nr; 1605 1606 if (phy_addr) 1607 *phy_addr = phy_base; 1608 1609 return cpu_base; 1610 } 1611