1 /* 2 * Copyright (c) 2016 Hisilicon Limited. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/platform_device.h> 35 #include "hns_roce_device.h" 36 #include "hns_roce_hem.h" 37 #include "hns_roce_common.h" 38 39 #define DMA_ADDR_T_SHIFT 12 40 #define BT_BA_SHIFT 32 41 42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) 43 { 44 if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) || 45 (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) || 46 (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) || 47 (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) || 48 (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) || 49 (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT)) 50 return true; 51 52 return false; 53 } 54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); 55 56 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, 57 u32 bt_chunk_num) 58 { 59 int i; 60 61 for (i = 0; i < bt_chunk_num; i++) 62 if (hem[start_idx + i]) 63 return false; 64 65 return true; 66 } 67 68 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num) 69 { 70 int i; 71 72 for (i = 0; i < bt_chunk_num; i++) 73 if (bt[start_idx + i]) 74 return false; 75 76 return true; 77 } 78 79 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num) 80 { 81 if (check_whether_bt_num_3(table_type, hop_num)) 82 return 3; 83 else if (check_whether_bt_num_2(table_type, hop_num)) 84 return 2; 85 else if (check_whether_bt_num_1(table_type, hop_num)) 86 return 1; 87 else 88 return 0; 89 } 90 91 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, 92 struct hns_roce_hem_table *table, unsigned long *obj, 93 struct hns_roce_hem_mhop *mhop) 94 { 95 struct device *dev = hr_dev->dev; 96 u32 chunk_ba_num; 97 u32 table_idx; 98 u32 bt_num; 99 u32 chunk_size; 100 101 switch (table->type) { 102 case HEM_TYPE_QPC: 103 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 104 + PAGE_SHIFT); 105 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 106 + PAGE_SHIFT); 107 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num; 108 mhop->hop_num = hr_dev->caps.qpc_hop_num; 109 break; 110 case HEM_TYPE_MTPT: 111 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 112 + PAGE_SHIFT); 113 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 114 + PAGE_SHIFT); 115 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num; 116 mhop->hop_num = hr_dev->caps.mpt_hop_num; 117 break; 118 case HEM_TYPE_CQC: 119 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 120 + PAGE_SHIFT); 121 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 122 + PAGE_SHIFT); 123 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num; 124 mhop->hop_num = hr_dev->caps.cqc_hop_num; 125 break; 126 case HEM_TYPE_SRQC: 127 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 128 + PAGE_SHIFT); 129 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 130 + PAGE_SHIFT); 131 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num; 132 mhop->hop_num = hr_dev->caps.srqc_hop_num; 133 break; 134 case HEM_TYPE_MTT: 135 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz 136 + PAGE_SHIFT); 137 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 138 + PAGE_SHIFT); 139 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 140 mhop->hop_num = hr_dev->caps.mtt_hop_num; 141 break; 142 case HEM_TYPE_CQE: 143 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz 144 + PAGE_SHIFT); 145 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 146 + PAGE_SHIFT); 147 mhop->ba_l0_num = mhop->bt_chunk_size / 8; 148 mhop->hop_num = hr_dev->caps.cqe_hop_num; 149 break; 150 default: 151 dev_err(dev, "Table %d not support multi-hop addressing!\n", 152 table->type); 153 return -EINVAL; 154 } 155 156 if (!obj) 157 return 0; 158 159 /* 160 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages. 161 * MTT/CQE alloc hem for bt pages. 162 */ 163 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); 164 chunk_ba_num = mhop->bt_chunk_size / 8; 165 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : 166 mhop->bt_chunk_size; 167 table_idx = (*obj & (table->num_obj - 1)) / 168 (chunk_size / table->obj_size); 169 switch (bt_num) { 170 case 3: 171 mhop->l2_idx = table_idx & (chunk_ba_num - 1); 172 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1); 173 mhop->l0_idx = table_idx / chunk_ba_num / chunk_ba_num; 174 break; 175 case 2: 176 mhop->l1_idx = table_idx & (chunk_ba_num - 1); 177 mhop->l0_idx = table_idx / chunk_ba_num; 178 break; 179 case 1: 180 mhop->l0_idx = table_idx; 181 break; 182 default: 183 dev_err(dev, "Table %d not support hop_num = %d!\n", 184 table->type, mhop->hop_num); 185 return -EINVAL; 186 } 187 if (mhop->l0_idx >= mhop->ba_l0_num) 188 mhop->l0_idx %= mhop->ba_l0_num; 189 190 return 0; 191 } 192 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); 193 194 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, 195 int npages, 196 unsigned long hem_alloc_size, 197 gfp_t gfp_mask) 198 { 199 struct hns_roce_hem_chunk *chunk = NULL; 200 struct hns_roce_hem *hem; 201 struct scatterlist *mem; 202 int order; 203 void *buf; 204 205 WARN_ON(gfp_mask & __GFP_HIGHMEM); 206 207 hem = kmalloc(sizeof(*hem), 208 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 209 if (!hem) 210 return NULL; 211 212 hem->refcount = 0; 213 INIT_LIST_HEAD(&hem->chunk_list); 214 215 order = get_order(hem_alloc_size); 216 217 while (npages > 0) { 218 if (!chunk) { 219 chunk = kmalloc(sizeof(*chunk), 220 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); 221 if (!chunk) 222 goto fail; 223 224 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN); 225 chunk->npages = 0; 226 chunk->nsg = 0; 227 list_add_tail(&chunk->list, &hem->chunk_list); 228 } 229 230 while (1 << order > npages) 231 --order; 232 233 /* 234 * Alloc memory one time. If failed, don't alloc small block 235 * memory, directly return fail. 236 */ 237 mem = &chunk->mem[chunk->npages]; 238 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order, 239 &sg_dma_address(mem), gfp_mask); 240 if (!buf) 241 goto fail; 242 243 sg_set_buf(mem, buf, PAGE_SIZE << order); 244 WARN_ON(mem->offset); 245 sg_dma_len(mem) = PAGE_SIZE << order; 246 247 ++chunk->npages; 248 ++chunk->nsg; 249 npages -= 1 << order; 250 } 251 252 return hem; 253 254 fail: 255 hns_roce_free_hem(hr_dev, hem); 256 return NULL; 257 } 258 259 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem) 260 { 261 struct hns_roce_hem_chunk *chunk, *tmp; 262 int i; 263 264 if (!hem) 265 return; 266 267 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) { 268 for (i = 0; i < chunk->npages; ++i) 269 dma_free_coherent(hr_dev->dev, 270 chunk->mem[i].length, 271 lowmem_page_address(sg_page(&chunk->mem[i])), 272 sg_dma_address(&chunk->mem[i])); 273 kfree(chunk); 274 } 275 276 kfree(hem); 277 } 278 279 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, 280 struct hns_roce_hem_table *table, unsigned long obj) 281 { 282 spinlock_t *lock = &hr_dev->bt_cmd_lock; 283 struct device *dev = hr_dev->dev; 284 unsigned long end = 0; 285 unsigned long flags; 286 struct hns_roce_hem_iter iter; 287 void __iomem *bt_cmd; 288 u32 bt_cmd_h_val = 0; 289 u32 bt_cmd_val[2]; 290 u32 bt_cmd_l = 0; 291 u64 bt_ba = 0; 292 int ret = 0; 293 294 /* Find the HEM(Hardware Entry Memory) entry */ 295 unsigned long i = (obj & (table->num_obj - 1)) / 296 (table->table_chunk_size / table->obj_size); 297 298 switch (table->type) { 299 case HEM_TYPE_QPC: 300 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 301 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC); 302 break; 303 case HEM_TYPE_MTPT: 304 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 305 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, 306 HEM_TYPE_MTPT); 307 break; 308 case HEM_TYPE_CQC: 309 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 310 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC); 311 break; 312 case HEM_TYPE_SRQC: 313 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M, 314 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, 315 HEM_TYPE_SRQC); 316 break; 317 default: 318 return ret; 319 } 320 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M, 321 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj); 322 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0); 323 roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1); 324 325 /* Currently iter only a chunk */ 326 for (hns_roce_hem_first(table->hem[i], &iter); 327 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) { 328 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT; 329 330 spin_lock_irqsave(lock, flags); 331 332 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; 333 334 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; 335 while (1) { 336 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { 337 if (!(time_before(jiffies, end))) { 338 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); 339 spin_unlock_irqrestore(lock, flags); 340 return -EBUSY; 341 } 342 } else { 343 break; 344 } 345 msleep(HW_SYNC_SLEEP_TIME_INTERVAL); 346 } 347 348 bt_cmd_l = (u32)bt_ba; 349 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M, 350 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, 351 bt_ba >> BT_BA_SHIFT); 352 353 bt_cmd_val[0] = bt_cmd_l; 354 bt_cmd_val[1] = bt_cmd_h_val; 355 hns_roce_write64_k(bt_cmd_val, 356 hr_dev->reg_base + ROCEE_BT_CMD_L_REG); 357 spin_unlock_irqrestore(lock, flags); 358 } 359 360 return ret; 361 } 362 363 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, 364 struct hns_roce_hem_table *table, 365 unsigned long obj) 366 { 367 struct device *dev = hr_dev->dev; 368 struct hns_roce_hem_mhop mhop; 369 struct hns_roce_hem_iter iter; 370 u32 buf_chunk_size; 371 u32 bt_chunk_size; 372 u32 chunk_ba_num; 373 u32 hop_num; 374 u32 size; 375 u32 bt_num; 376 u64 hem_idx; 377 u64 bt_l1_idx = 0; 378 u64 bt_l0_idx = 0; 379 u64 bt_ba; 380 unsigned long mhop_obj = obj; 381 int bt_l1_allocated = 0; 382 int bt_l0_allocated = 0; 383 int step_idx; 384 int ret; 385 386 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 387 if (ret) 388 return ret; 389 390 buf_chunk_size = mhop.buf_chunk_size; 391 bt_chunk_size = mhop.bt_chunk_size; 392 hop_num = mhop.hop_num; 393 chunk_ba_num = bt_chunk_size / 8; 394 395 bt_num = hns_roce_get_bt_num(table->type, hop_num); 396 switch (bt_num) { 397 case 3: 398 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 399 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 400 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 401 bt_l0_idx = mhop.l0_idx; 402 break; 403 case 2: 404 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 405 bt_l0_idx = mhop.l0_idx; 406 break; 407 case 1: 408 hem_idx = mhop.l0_idx; 409 break; 410 default: 411 dev_err(dev, "Table %d not support hop_num = %d!\n", 412 table->type, hop_num); 413 return -EINVAL; 414 } 415 416 mutex_lock(&table->mutex); 417 418 if (table->hem[hem_idx]) { 419 ++table->hem[hem_idx]->refcount; 420 goto out; 421 } 422 423 /* alloc L1 BA's chunk */ 424 if ((check_whether_bt_num_3(table->type, hop_num) || 425 check_whether_bt_num_2(table->type, hop_num)) && 426 !table->bt_l0[bt_l0_idx]) { 427 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size, 428 &(table->bt_l0_dma_addr[bt_l0_idx]), 429 GFP_KERNEL); 430 if (!table->bt_l0[bt_l0_idx]) { 431 ret = -ENOMEM; 432 goto out; 433 } 434 bt_l0_allocated = 1; 435 436 /* set base address to hardware */ 437 if (table->type < HEM_TYPE_MTT) { 438 step_idx = 0; 439 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 440 ret = -ENODEV; 441 dev_err(dev, "set HEM base address to HW failed!\n"); 442 goto err_dma_alloc_l1; 443 } 444 } 445 } 446 447 /* alloc L2 BA's chunk */ 448 if (check_whether_bt_num_3(table->type, hop_num) && 449 !table->bt_l1[bt_l1_idx]) { 450 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size, 451 &(table->bt_l1_dma_addr[bt_l1_idx]), 452 GFP_KERNEL); 453 if (!table->bt_l1[bt_l1_idx]) { 454 ret = -ENOMEM; 455 goto err_dma_alloc_l1; 456 } 457 bt_l1_allocated = 1; 458 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = 459 table->bt_l1_dma_addr[bt_l1_idx]; 460 461 /* set base address to hardware */ 462 step_idx = 1; 463 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 464 ret = -ENODEV; 465 dev_err(dev, "set HEM base address to HW failed!\n"); 466 goto err_alloc_hem_buf; 467 } 468 } 469 470 /* 471 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC. 472 * alloc bt space chunk for MTT/CQE. 473 */ 474 size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size; 475 table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev, 476 size >> PAGE_SHIFT, 477 size, 478 (table->lowmem ? GFP_KERNEL : 479 GFP_HIGHUSER) | __GFP_NOWARN); 480 if (!table->hem[hem_idx]) { 481 ret = -ENOMEM; 482 goto err_alloc_hem_buf; 483 } 484 485 hns_roce_hem_first(table->hem[hem_idx], &iter); 486 bt_ba = hns_roce_hem_addr(&iter); 487 488 if (table->type < HEM_TYPE_MTT) { 489 if (hop_num == 2) { 490 *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba; 491 step_idx = 2; 492 } else if (hop_num == 1) { 493 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 494 step_idx = 1; 495 } else if (hop_num == HNS_ROCE_HOP_NUM_0) { 496 step_idx = 0; 497 } 498 499 /* set HEM base address to hardware */ 500 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) { 501 ret = -ENODEV; 502 dev_err(dev, "set HEM base address to HW failed!\n"); 503 goto err_alloc_hem_buf; 504 } 505 } else if (hop_num == 2) { 506 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba; 507 } 508 509 ++table->hem[hem_idx]->refcount; 510 goto out; 511 512 err_alloc_hem_buf: 513 if (bt_l1_allocated) { 514 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx], 515 table->bt_l1_dma_addr[bt_l1_idx]); 516 table->bt_l1[bt_l1_idx] = NULL; 517 } 518 519 err_dma_alloc_l1: 520 if (bt_l0_allocated) { 521 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx], 522 table->bt_l0_dma_addr[bt_l0_idx]); 523 table->bt_l0[bt_l0_idx] = NULL; 524 } 525 526 out: 527 mutex_unlock(&table->mutex); 528 return ret; 529 } 530 531 int hns_roce_table_get(struct hns_roce_dev *hr_dev, 532 struct hns_roce_hem_table *table, unsigned long obj) 533 { 534 struct device *dev = hr_dev->dev; 535 int ret = 0; 536 unsigned long i; 537 538 if (hns_roce_check_whether_mhop(hr_dev, table->type)) 539 return hns_roce_table_mhop_get(hr_dev, table, obj); 540 541 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size / 542 table->obj_size); 543 544 mutex_lock(&table->mutex); 545 546 if (table->hem[i]) { 547 ++table->hem[i]->refcount; 548 goto out; 549 } 550 551 table->hem[i] = hns_roce_alloc_hem(hr_dev, 552 table->table_chunk_size >> PAGE_SHIFT, 553 table->table_chunk_size, 554 (table->lowmem ? GFP_KERNEL : 555 GFP_HIGHUSER) | __GFP_NOWARN); 556 if (!table->hem[i]) { 557 ret = -ENOMEM; 558 goto out; 559 } 560 561 /* Set HEM base address(128K/page, pa) to Hardware */ 562 if (hns_roce_set_hem(hr_dev, table, obj)) { 563 hns_roce_free_hem(hr_dev, table->hem[i]); 564 table->hem[i] = NULL; 565 ret = -ENODEV; 566 dev_err(dev, "set HEM base address to HW failed.\n"); 567 goto out; 568 } 569 570 ++table->hem[i]->refcount; 571 out: 572 mutex_unlock(&table->mutex); 573 return ret; 574 } 575 576 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, 577 struct hns_roce_hem_table *table, 578 unsigned long obj, 579 int check_refcount) 580 { 581 struct device *dev = hr_dev->dev; 582 struct hns_roce_hem_mhop mhop; 583 unsigned long mhop_obj = obj; 584 u32 bt_chunk_size; 585 u32 chunk_ba_num; 586 u32 hop_num; 587 u32 start_idx; 588 u32 bt_num; 589 u64 hem_idx; 590 u64 bt_l1_idx = 0; 591 int ret; 592 593 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 594 if (ret) 595 return; 596 597 bt_chunk_size = mhop.bt_chunk_size; 598 hop_num = mhop.hop_num; 599 chunk_ba_num = bt_chunk_size / 8; 600 601 bt_num = hns_roce_get_bt_num(table->type, hop_num); 602 switch (bt_num) { 603 case 3: 604 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 605 mhop.l1_idx * chunk_ba_num + mhop.l2_idx; 606 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 607 break; 608 case 2: 609 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx; 610 break; 611 case 1: 612 hem_idx = mhop.l0_idx; 613 break; 614 default: 615 dev_err(dev, "Table %d not support hop_num = %d!\n", 616 table->type, hop_num); 617 return; 618 } 619 620 mutex_lock(&table->mutex); 621 622 if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) { 623 mutex_unlock(&table->mutex); 624 return; 625 } 626 627 if (table->type < HEM_TYPE_MTT && hop_num == 1) { 628 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 629 dev_warn(dev, "Clear HEM base address failed.\n"); 630 } else if (table->type < HEM_TYPE_MTT && hop_num == 2) { 631 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2)) 632 dev_warn(dev, "Clear HEM base address failed.\n"); 633 } else if (table->type < HEM_TYPE_MTT && 634 hop_num == HNS_ROCE_HOP_NUM_0) { 635 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 636 dev_warn(dev, "Clear HEM base address failed.\n"); 637 } 638 639 /* 640 * free buffer space chunk for QPC/MTPT/CQC/SRQC. 641 * free bt space chunk for MTT/CQE. 642 */ 643 hns_roce_free_hem(hr_dev, table->hem[hem_idx]); 644 table->hem[hem_idx] = NULL; 645 646 if (check_whether_bt_num_2(table->type, hop_num)) { 647 start_idx = mhop.l0_idx * chunk_ba_num; 648 if (hns_roce_check_hem_null(table->hem, start_idx, 649 chunk_ba_num)) { 650 if (table->type < HEM_TYPE_MTT && 651 hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 652 dev_warn(dev, "Clear HEM base address failed.\n"); 653 654 dma_free_coherent(dev, bt_chunk_size, 655 table->bt_l0[mhop.l0_idx], 656 table->bt_l0_dma_addr[mhop.l0_idx]); 657 table->bt_l0[mhop.l0_idx] = NULL; 658 } 659 } else if (check_whether_bt_num_3(table->type, hop_num)) { 660 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num + 661 mhop.l1_idx * chunk_ba_num; 662 if (hns_roce_check_hem_null(table->hem, start_idx, 663 chunk_ba_num)) { 664 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1)) 665 dev_warn(dev, "Clear HEM base address failed.\n"); 666 667 dma_free_coherent(dev, bt_chunk_size, 668 table->bt_l1[bt_l1_idx], 669 table->bt_l1_dma_addr[bt_l1_idx]); 670 table->bt_l1[bt_l1_idx] = NULL; 671 672 start_idx = mhop.l0_idx * chunk_ba_num; 673 if (hns_roce_check_bt_null(table->bt_l1, start_idx, 674 chunk_ba_num)) { 675 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 676 0)) 677 dev_warn(dev, "Clear HEM base address failed.\n"); 678 679 dma_free_coherent(dev, bt_chunk_size, 680 table->bt_l0[mhop.l0_idx], 681 table->bt_l0_dma_addr[mhop.l0_idx]); 682 table->bt_l0[mhop.l0_idx] = NULL; 683 } 684 } 685 } 686 687 mutex_unlock(&table->mutex); 688 } 689 690 void hns_roce_table_put(struct hns_roce_dev *hr_dev, 691 struct hns_roce_hem_table *table, unsigned long obj) 692 { 693 struct device *dev = hr_dev->dev; 694 unsigned long i; 695 696 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 697 hns_roce_table_mhop_put(hr_dev, table, obj, 1); 698 return; 699 } 700 701 i = (obj & (table->num_obj - 1)) / 702 (table->table_chunk_size / table->obj_size); 703 704 mutex_lock(&table->mutex); 705 706 if (--table->hem[i]->refcount == 0) { 707 /* Clear HEM base address */ 708 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0)) 709 dev_warn(dev, "Clear HEM base address failed.\n"); 710 711 hns_roce_free_hem(hr_dev, table->hem[i]); 712 table->hem[i] = NULL; 713 } 714 715 mutex_unlock(&table->mutex); 716 } 717 718 void *hns_roce_table_find(struct hns_roce_dev *hr_dev, 719 struct hns_roce_hem_table *table, 720 unsigned long obj, dma_addr_t *dma_handle) 721 { 722 struct hns_roce_hem_chunk *chunk; 723 struct hns_roce_hem_mhop mhop; 724 struct hns_roce_hem *hem; 725 struct page *page = NULL; 726 unsigned long mhop_obj = obj; 727 unsigned long obj_per_chunk; 728 unsigned long idx_offset; 729 int offset, dma_offset; 730 int i, j; 731 u32 hem_idx = 0; 732 733 if (!table->lowmem) 734 return NULL; 735 736 mutex_lock(&table->mutex); 737 738 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) { 739 obj_per_chunk = table->table_chunk_size / table->obj_size; 740 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk]; 741 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk; 742 dma_offset = offset = idx_offset * table->obj_size; 743 } else { 744 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop); 745 /* mtt mhop */ 746 i = mhop.l0_idx; 747 j = mhop.l1_idx; 748 if (mhop.hop_num == 2) 749 hem_idx = i * (mhop.bt_chunk_size / 8) + j; 750 else if (mhop.hop_num == 1 || 751 mhop.hop_num == HNS_ROCE_HOP_NUM_0) 752 hem_idx = i; 753 754 hem = table->hem[hem_idx]; 755 dma_offset = offset = (obj & (table->num_obj - 1)) * 756 table->obj_size % mhop.bt_chunk_size; 757 if (mhop.hop_num == 2) 758 dma_offset = offset = 0; 759 } 760 761 if (!hem) 762 goto out; 763 764 list_for_each_entry(chunk, &hem->chunk_list, list) { 765 for (i = 0; i < chunk->npages; ++i) { 766 if (dma_handle && dma_offset >= 0) { 767 if (sg_dma_len(&chunk->mem[i]) > 768 (u32)dma_offset) 769 *dma_handle = sg_dma_address( 770 &chunk->mem[i]) + dma_offset; 771 dma_offset -= sg_dma_len(&chunk->mem[i]); 772 } 773 774 if (chunk->mem[i].length > (u32)offset) { 775 page = sg_page(&chunk->mem[i]); 776 goto out; 777 } 778 offset -= chunk->mem[i].length; 779 } 780 } 781 782 out: 783 mutex_unlock(&table->mutex); 784 return page ? lowmem_page_address(page) + offset : NULL; 785 } 786 EXPORT_SYMBOL_GPL(hns_roce_table_find); 787 788 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, 789 struct hns_roce_hem_table *table, 790 unsigned long start, unsigned long end) 791 { 792 struct hns_roce_hem_mhop mhop; 793 unsigned long inc = table->table_chunk_size / table->obj_size; 794 unsigned long i; 795 int ret; 796 797 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 798 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 799 inc = mhop.bt_chunk_size / table->obj_size; 800 } 801 802 /* Allocate MTT entry memory according to chunk(128K) */ 803 for (i = start; i <= end; i += inc) { 804 ret = hns_roce_table_get(hr_dev, table, i); 805 if (ret) 806 goto fail; 807 } 808 809 return 0; 810 811 fail: 812 while (i > start) { 813 i -= inc; 814 hns_roce_table_put(hr_dev, table, i); 815 } 816 return ret; 817 } 818 819 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev, 820 struct hns_roce_hem_table *table, 821 unsigned long start, unsigned long end) 822 { 823 struct hns_roce_hem_mhop mhop; 824 unsigned long inc = table->table_chunk_size / table->obj_size; 825 unsigned long i; 826 827 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 828 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 829 inc = mhop.bt_chunk_size / table->obj_size; 830 } 831 832 for (i = start; i <= end; i += inc) 833 hns_roce_table_put(hr_dev, table, i); 834 } 835 836 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, 837 struct hns_roce_hem_table *table, u32 type, 838 unsigned long obj_size, unsigned long nobj, 839 int use_lowmem) 840 { 841 struct device *dev = hr_dev->dev; 842 unsigned long obj_per_chunk; 843 unsigned long num_hem; 844 845 if (!hns_roce_check_whether_mhop(hr_dev, type)) { 846 table->table_chunk_size = hr_dev->caps.chunk_sz; 847 obj_per_chunk = table->table_chunk_size / obj_size; 848 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 849 850 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL); 851 if (!table->hem) 852 return -ENOMEM; 853 } else { 854 unsigned long buf_chunk_size; 855 unsigned long bt_chunk_size; 856 unsigned long bt_chunk_num; 857 unsigned long num_bt_l0 = 0; 858 u32 hop_num; 859 860 switch (type) { 861 case HEM_TYPE_QPC: 862 buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz 863 + PAGE_SHIFT); 864 bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz 865 + PAGE_SHIFT); 866 num_bt_l0 = hr_dev->caps.qpc_bt_num; 867 hop_num = hr_dev->caps.qpc_hop_num; 868 break; 869 case HEM_TYPE_MTPT: 870 buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz 871 + PAGE_SHIFT); 872 bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz 873 + PAGE_SHIFT); 874 num_bt_l0 = hr_dev->caps.mpt_bt_num; 875 hop_num = hr_dev->caps.mpt_hop_num; 876 break; 877 case HEM_TYPE_CQC: 878 buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz 879 + PAGE_SHIFT); 880 bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz 881 + PAGE_SHIFT); 882 num_bt_l0 = hr_dev->caps.cqc_bt_num; 883 hop_num = hr_dev->caps.cqc_hop_num; 884 break; 885 case HEM_TYPE_SRQC: 886 buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz 887 + PAGE_SHIFT); 888 bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz 889 + PAGE_SHIFT); 890 num_bt_l0 = hr_dev->caps.srqc_bt_num; 891 hop_num = hr_dev->caps.srqc_hop_num; 892 break; 893 case HEM_TYPE_MTT: 894 buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz 895 + PAGE_SHIFT); 896 bt_chunk_size = buf_chunk_size; 897 hop_num = hr_dev->caps.mtt_hop_num; 898 break; 899 case HEM_TYPE_CQE: 900 buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz 901 + PAGE_SHIFT); 902 bt_chunk_size = buf_chunk_size; 903 hop_num = hr_dev->caps.cqe_hop_num; 904 break; 905 default: 906 dev_err(dev, 907 "Table %d not support to init hem table here!\n", 908 type); 909 return -EINVAL; 910 } 911 obj_per_chunk = buf_chunk_size / obj_size; 912 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; 913 bt_chunk_num = bt_chunk_size / 8; 914 if (table->type >= HEM_TYPE_MTT) 915 num_bt_l0 = bt_chunk_num; 916 917 table->hem = kcalloc(num_hem, sizeof(*table->hem), 918 GFP_KERNEL); 919 if (!table->hem) 920 goto err_kcalloc_hem_buf; 921 922 if (check_whether_bt_num_3(table->type, hop_num)) { 923 unsigned long num_bt_l1; 924 925 num_bt_l1 = (num_hem + bt_chunk_num - 1) / 926 bt_chunk_num; 927 table->bt_l1 = kcalloc(num_bt_l1, 928 sizeof(*table->bt_l1), 929 GFP_KERNEL); 930 if (!table->bt_l1) 931 goto err_kcalloc_bt_l1; 932 933 table->bt_l1_dma_addr = kcalloc(num_bt_l1, 934 sizeof(*table->bt_l1_dma_addr), 935 GFP_KERNEL); 936 937 if (!table->bt_l1_dma_addr) 938 goto err_kcalloc_l1_dma; 939 } 940 941 if (check_whether_bt_num_2(table->type, hop_num) || 942 check_whether_bt_num_3(table->type, hop_num)) { 943 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0), 944 GFP_KERNEL); 945 if (!table->bt_l0) 946 goto err_kcalloc_bt_l0; 947 948 table->bt_l0_dma_addr = kcalloc(num_bt_l0, 949 sizeof(*table->bt_l0_dma_addr), 950 GFP_KERNEL); 951 if (!table->bt_l0_dma_addr) 952 goto err_kcalloc_l0_dma; 953 } 954 } 955 956 table->type = type; 957 table->num_hem = num_hem; 958 table->num_obj = nobj; 959 table->obj_size = obj_size; 960 table->lowmem = use_lowmem; 961 mutex_init(&table->mutex); 962 963 return 0; 964 965 err_kcalloc_l0_dma: 966 kfree(table->bt_l0); 967 table->bt_l0 = NULL; 968 969 err_kcalloc_bt_l0: 970 kfree(table->bt_l1_dma_addr); 971 table->bt_l1_dma_addr = NULL; 972 973 err_kcalloc_l1_dma: 974 kfree(table->bt_l1); 975 table->bt_l1 = NULL; 976 977 err_kcalloc_bt_l1: 978 kfree(table->hem); 979 table->hem = NULL; 980 981 err_kcalloc_hem_buf: 982 return -ENOMEM; 983 } 984 985 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev, 986 struct hns_roce_hem_table *table) 987 { 988 struct hns_roce_hem_mhop mhop; 989 u32 buf_chunk_size; 990 int i; 991 u64 obj; 992 993 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop); 994 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size : 995 mhop.bt_chunk_size; 996 997 for (i = 0; i < table->num_hem; ++i) { 998 obj = i * buf_chunk_size / table->obj_size; 999 if (table->hem[i]) 1000 hns_roce_table_mhop_put(hr_dev, table, obj, 0); 1001 } 1002 1003 kfree(table->hem); 1004 table->hem = NULL; 1005 kfree(table->bt_l1); 1006 table->bt_l1 = NULL; 1007 kfree(table->bt_l1_dma_addr); 1008 table->bt_l1_dma_addr = NULL; 1009 kfree(table->bt_l0); 1010 table->bt_l0 = NULL; 1011 kfree(table->bt_l0_dma_addr); 1012 table->bt_l0_dma_addr = NULL; 1013 } 1014 1015 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev, 1016 struct hns_roce_hem_table *table) 1017 { 1018 struct device *dev = hr_dev->dev; 1019 unsigned long i; 1020 1021 if (hns_roce_check_whether_mhop(hr_dev, table->type)) { 1022 hns_roce_cleanup_mhop_hem_table(hr_dev, table); 1023 return; 1024 } 1025 1026 for (i = 0; i < table->num_hem; ++i) 1027 if (table->hem[i]) { 1028 if (hr_dev->hw->clear_hem(hr_dev, table, 1029 i * table->table_chunk_size / table->obj_size, 0)) 1030 dev_err(dev, "Clear HEM base address failed.\n"); 1031 1032 hns_roce_free_hem(hr_dev, table->hem[i]); 1033 } 1034 1035 kfree(table->hem); 1036 } 1037 1038 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) 1039 { 1040 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table); 1041 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table); 1042 if (hr_dev->caps.trrl_entry_sz) 1043 hns_roce_cleanup_hem_table(hr_dev, 1044 &hr_dev->qp_table.trrl_table); 1045 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table); 1046 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table); 1047 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); 1048 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) 1049 hns_roce_cleanup_hem_table(hr_dev, 1050 &hr_dev->mr_table.mtt_cqe_table); 1051 } 1052