1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved. 4 */ 5 6 #include "mana_ib.h" 7 8 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, 9 u32 port) 10 { 11 struct gdma_dev *gd = dev->gdma_dev; 12 struct mana_port_context *mpc; 13 struct net_device *ndev; 14 struct mana_context *mc; 15 16 mc = gd->driver_data; 17 ndev = mc->ports[port]; 18 mpc = netdev_priv(ndev); 19 20 mutex_lock(&pd->vport_mutex); 21 22 pd->vport_use_count--; 23 WARN_ON(pd->vport_use_count < 0); 24 25 if (!pd->vport_use_count) 26 mana_uncfg_vport(mpc); 27 28 mutex_unlock(&pd->vport_mutex); 29 } 30 31 int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd, 32 u32 doorbell_id) 33 { 34 struct gdma_dev *mdev = dev->gdma_dev; 35 struct mana_port_context *mpc; 36 struct mana_context *mc; 37 struct net_device *ndev; 38 int err; 39 40 mc = mdev->driver_data; 41 ndev = mc->ports[port]; 42 mpc = netdev_priv(ndev); 43 44 mutex_lock(&pd->vport_mutex); 45 46 pd->vport_use_count++; 47 if (pd->vport_use_count > 1) { 48 ibdev_dbg(&dev->ib_dev, 49 "Skip as this PD is already configured vport\n"); 50 mutex_unlock(&pd->vport_mutex); 51 return 0; 52 } 53 54 err = mana_cfg_vport(mpc, pd->pdn, doorbell_id); 55 if (err) { 56 pd->vport_use_count--; 57 mutex_unlock(&pd->vport_mutex); 58 59 ibdev_dbg(&dev->ib_dev, "Failed to configure vPort %d\n", err); 60 return err; 61 } 62 63 mutex_unlock(&pd->vport_mutex); 64 65 pd->tx_shortform_allowed = mpc->tx_shortform_allowed; 66 pd->tx_vp_offset = mpc->tx_vp_offset; 67 68 ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n", 69 mpc->port_handle, pd->pdn, doorbell_id); 70 71 return 0; 72 } 73 74 int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 75 { 76 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 77 struct ib_device *ibdev = ibpd->device; 78 struct gdma_create_pd_resp resp = {}; 79 struct gdma_create_pd_req req = {}; 80 enum gdma_pd_flags flags = 0; 81 struct mana_ib_dev *dev; 82 struct gdma_dev *mdev; 83 int err; 84 85 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 86 mdev = dev->gdma_dev; 87 88 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req), 89 sizeof(resp)); 90 91 req.flags = flags; 92 err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req, 93 sizeof(resp), &resp); 94 95 if (err || resp.hdr.status) { 96 ibdev_dbg(&dev->ib_dev, 97 "Failed to get pd_id err %d status %u\n", err, 98 resp.hdr.status); 99 if (!err) 100 err = -EPROTO; 101 102 return err; 103 } 104 105 pd->pd_handle = resp.pd_handle; 106 pd->pdn = resp.pd_id; 107 ibdev_dbg(&dev->ib_dev, "pd_handle 0x%llx pd_id %d\n", 108 pd->pd_handle, pd->pdn); 109 110 mutex_init(&pd->vport_mutex); 111 pd->vport_use_count = 0; 112 return 0; 113 } 114 115 int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 116 { 117 struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd); 118 struct ib_device *ibdev = ibpd->device; 119 struct gdma_destory_pd_resp resp = {}; 120 struct gdma_destroy_pd_req req = {}; 121 struct mana_ib_dev *dev; 122 struct gdma_dev *mdev; 123 int err; 124 125 dev = container_of(ibdev, struct mana_ib_dev, ib_dev); 126 mdev = dev->gdma_dev; 127 128 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req), 129 sizeof(resp)); 130 131 req.pd_handle = pd->pd_handle; 132 err = mana_gd_send_request(mdev->gdma_context, sizeof(req), &req, 133 sizeof(resp), &resp); 134 135 if (err || resp.hdr.status) { 136 ibdev_dbg(&dev->ib_dev, 137 "Failed to destroy pd_handle 0x%llx err %d status %u", 138 pd->pd_handle, err, resp.hdr.status); 139 if (!err) 140 err = -EPROTO; 141 } 142 143 return err; 144 } 145 146 static int mana_gd_destroy_doorbell_page(struct gdma_context *gc, 147 int doorbell_page) 148 { 149 struct gdma_destroy_resource_range_req req = {}; 150 struct gdma_resp_hdr resp = {}; 151 int err; 152 153 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_RESOURCE_RANGE, 154 sizeof(req), sizeof(resp)); 155 156 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE; 157 req.num_resources = 1; 158 req.allocated_resources = doorbell_page; 159 160 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 161 if (err || resp.status) { 162 dev_err(gc->dev, 163 "Failed to destroy doorbell page: ret %d, 0x%x\n", 164 err, resp.status); 165 return err ?: -EPROTO; 166 } 167 168 return 0; 169 } 170 171 static int mana_gd_allocate_doorbell_page(struct gdma_context *gc, 172 int *doorbell_page) 173 { 174 struct gdma_allocate_resource_range_req req = {}; 175 struct gdma_allocate_resource_range_resp resp = {}; 176 int err; 177 178 mana_gd_init_req_hdr(&req.hdr, GDMA_ALLOCATE_RESOURCE_RANGE, 179 sizeof(req), sizeof(resp)); 180 181 req.resource_type = GDMA_RESOURCE_DOORBELL_PAGE; 182 req.num_resources = 1; 183 req.alignment = 1; 184 185 /* Have GDMA start searching from 0 */ 186 req.allocated_resources = 0; 187 188 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 189 if (err || resp.hdr.status) { 190 dev_err(gc->dev, 191 "Failed to allocate doorbell page: ret %d, 0x%x\n", 192 err, resp.hdr.status); 193 return err ?: -EPROTO; 194 } 195 196 *doorbell_page = resp.allocated_resources; 197 198 return 0; 199 } 200 201 int mana_ib_alloc_ucontext(struct ib_ucontext *ibcontext, 202 struct ib_udata *udata) 203 { 204 struct mana_ib_ucontext *ucontext = 205 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 206 struct ib_device *ibdev = ibcontext->device; 207 struct mana_ib_dev *mdev; 208 struct gdma_context *gc; 209 struct gdma_dev *dev; 210 int doorbell_page; 211 int ret; 212 213 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 214 dev = mdev->gdma_dev; 215 gc = dev->gdma_context; 216 217 /* Allocate a doorbell page index */ 218 ret = mana_gd_allocate_doorbell_page(gc, &doorbell_page); 219 if (ret) { 220 ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret); 221 return ret; 222 } 223 224 ibdev_dbg(ibdev, "Doorbell page allocated %d\n", doorbell_page); 225 226 ucontext->doorbell = doorbell_page; 227 228 return 0; 229 } 230 231 void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcontext) 232 { 233 struct mana_ib_ucontext *mana_ucontext = 234 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 235 struct ib_device *ibdev = ibcontext->device; 236 struct mana_ib_dev *mdev; 237 struct gdma_context *gc; 238 int ret; 239 240 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 241 gc = mdev->gdma_dev->gdma_context; 242 243 ret = mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell); 244 if (ret) 245 ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret); 246 } 247 248 static int 249 mana_ib_gd_first_dma_region(struct mana_ib_dev *dev, 250 struct gdma_context *gc, 251 struct gdma_create_dma_region_req *create_req, 252 size_t num_pages, mana_handle_t *gdma_region) 253 { 254 struct gdma_create_dma_region_resp create_resp = {}; 255 unsigned int create_req_msg_size; 256 int err; 257 258 create_req_msg_size = 259 struct_size(create_req, page_addr_list, num_pages); 260 create_req->page_addr_list_len = num_pages; 261 262 err = mana_gd_send_request(gc, create_req_msg_size, create_req, 263 sizeof(create_resp), &create_resp); 264 if (err || create_resp.hdr.status) { 265 ibdev_dbg(&dev->ib_dev, 266 "Failed to create DMA region: %d, 0x%x\n", 267 err, create_resp.hdr.status); 268 if (!err) 269 err = -EPROTO; 270 271 return err; 272 } 273 274 *gdma_region = create_resp.dma_region_handle; 275 ibdev_dbg(&dev->ib_dev, "Created DMA region handle 0x%llx\n", 276 *gdma_region); 277 278 return 0; 279 } 280 281 static int 282 mana_ib_gd_add_dma_region(struct mana_ib_dev *dev, struct gdma_context *gc, 283 struct gdma_dma_region_add_pages_req *add_req, 284 unsigned int num_pages, u32 expected_status) 285 { 286 unsigned int add_req_msg_size = 287 struct_size(add_req, page_addr_list, num_pages); 288 struct gdma_general_resp add_resp = {}; 289 int err; 290 291 mana_gd_init_req_hdr(&add_req->hdr, GDMA_DMA_REGION_ADD_PAGES, 292 add_req_msg_size, sizeof(add_resp)); 293 add_req->page_addr_list_len = num_pages; 294 295 err = mana_gd_send_request(gc, add_req_msg_size, add_req, 296 sizeof(add_resp), &add_resp); 297 if (err || add_resp.hdr.status != expected_status) { 298 ibdev_dbg(&dev->ib_dev, 299 "Failed to create DMA region: %d, 0x%x\n", 300 err, add_resp.hdr.status); 301 302 if (!err) 303 err = -EPROTO; 304 305 return err; 306 } 307 308 return 0; 309 } 310 311 int mana_ib_gd_create_dma_region(struct mana_ib_dev *dev, struct ib_umem *umem, 312 mana_handle_t *gdma_region) 313 { 314 struct gdma_dma_region_add_pages_req *add_req = NULL; 315 size_t num_pages_processed = 0, num_pages_to_handle; 316 struct gdma_create_dma_region_req *create_req; 317 unsigned int create_req_msg_size; 318 struct hw_channel_context *hwc; 319 struct ib_block_iter biter; 320 size_t max_pgs_add_cmd = 0; 321 size_t max_pgs_create_cmd; 322 struct gdma_context *gc; 323 size_t num_pages_total; 324 struct gdma_dev *mdev; 325 unsigned long page_sz; 326 unsigned int tail = 0; 327 u64 *page_addr_list; 328 void *request_buf; 329 int err; 330 331 mdev = dev->gdma_dev; 332 gc = mdev->gdma_context; 333 hwc = gc->hwc.driver_data; 334 335 /* Hardware requires dma region to align to chosen page size */ 336 page_sz = ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, 0); 337 if (!page_sz) { 338 ibdev_dbg(&dev->ib_dev, "failed to find page size.\n"); 339 return -ENOMEM; 340 } 341 num_pages_total = ib_umem_num_dma_blocks(umem, page_sz); 342 343 max_pgs_create_cmd = 344 (hwc->max_req_msg_size - sizeof(*create_req)) / sizeof(u64); 345 num_pages_to_handle = 346 min_t(size_t, num_pages_total, max_pgs_create_cmd); 347 create_req_msg_size = 348 struct_size(create_req, page_addr_list, num_pages_to_handle); 349 350 request_buf = kzalloc(hwc->max_req_msg_size, GFP_KERNEL); 351 if (!request_buf) 352 return -ENOMEM; 353 354 create_req = request_buf; 355 mana_gd_init_req_hdr(&create_req->hdr, GDMA_CREATE_DMA_REGION, 356 create_req_msg_size, 357 sizeof(struct gdma_create_dma_region_resp)); 358 359 create_req->length = umem->length; 360 create_req->offset_in_page = umem->address & (page_sz - 1); 361 create_req->gdma_page_type = order_base_2(page_sz) - PAGE_SHIFT; 362 create_req->page_count = num_pages_total; 363 364 ibdev_dbg(&dev->ib_dev, "size_dma_region %lu num_pages_total %lu\n", 365 umem->length, num_pages_total); 366 367 ibdev_dbg(&dev->ib_dev, "page_sz %lu offset_in_page %u\n", 368 page_sz, create_req->offset_in_page); 369 370 ibdev_dbg(&dev->ib_dev, "num_pages_to_handle %lu, gdma_page_type %u", 371 num_pages_to_handle, create_req->gdma_page_type); 372 373 page_addr_list = create_req->page_addr_list; 374 rdma_umem_for_each_dma_block(umem, &biter, page_sz) { 375 page_addr_list[tail++] = rdma_block_iter_dma_address(&biter); 376 if (tail < num_pages_to_handle) 377 continue; 378 379 if (!num_pages_processed) { 380 /* First create message */ 381 err = mana_ib_gd_first_dma_region(dev, gc, create_req, 382 tail, gdma_region); 383 if (err) 384 goto out; 385 386 max_pgs_add_cmd = (hwc->max_req_msg_size - 387 sizeof(*add_req)) / sizeof(u64); 388 389 add_req = request_buf; 390 add_req->dma_region_handle = *gdma_region; 391 add_req->reserved3 = 0; 392 page_addr_list = add_req->page_addr_list; 393 } else { 394 /* Subsequent create messages */ 395 u32 expected_s = 0; 396 397 if (num_pages_processed + num_pages_to_handle < 398 num_pages_total) 399 expected_s = GDMA_STATUS_MORE_ENTRIES; 400 401 err = mana_ib_gd_add_dma_region(dev, gc, add_req, tail, 402 expected_s); 403 if (err) 404 break; 405 } 406 407 num_pages_processed += tail; 408 tail = 0; 409 410 /* The remaining pages to create */ 411 num_pages_to_handle = 412 min_t(size_t, 413 num_pages_total - num_pages_processed, 414 max_pgs_add_cmd); 415 } 416 417 if (err) 418 mana_ib_gd_destroy_dma_region(dev, *gdma_region); 419 420 out: 421 kfree(request_buf); 422 return err; 423 } 424 425 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *dev, u64 gdma_region) 426 { 427 struct gdma_dev *mdev = dev->gdma_dev; 428 struct gdma_context *gc; 429 430 gc = mdev->gdma_context; 431 ibdev_dbg(&dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region); 432 433 return mana_gd_destroy_dma_region(gc, gdma_region); 434 } 435 436 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) 437 { 438 struct mana_ib_ucontext *mana_ucontext = 439 container_of(ibcontext, struct mana_ib_ucontext, ibucontext); 440 struct ib_device *ibdev = ibcontext->device; 441 struct mana_ib_dev *mdev; 442 struct gdma_context *gc; 443 phys_addr_t pfn; 444 pgprot_t prot; 445 int ret; 446 447 mdev = container_of(ibdev, struct mana_ib_dev, ib_dev); 448 gc = mdev->gdma_dev->gdma_context; 449 450 if (vma->vm_pgoff != 0) { 451 ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff); 452 return -EINVAL; 453 } 454 455 /* Map to the page indexed by ucontext->doorbell */ 456 pfn = (gc->phys_db_page_base + 457 gc->db_page_size * mana_ucontext->doorbell) >> 458 PAGE_SHIFT; 459 prot = pgprot_writecombine(vma->vm_page_prot); 460 461 ret = rdma_user_mmap_io(ibcontext, vma, pfn, gc->db_page_size, prot, 462 NULL); 463 if (ret) 464 ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret); 465 else 466 ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %u, ret %d\n", 467 pfn, gc->db_page_size, ret); 468 469 return ret; 470 } 471 472 int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, 473 struct ib_port_immutable *immutable) 474 { 475 /* 476 * This version only support RAW_PACKET 477 * other values need to be filled for other types 478 */ 479 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET; 480 481 return 0; 482 } 483 484 int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *props, 485 struct ib_udata *uhw) 486 { 487 props->max_qp = MANA_MAX_NUM_QUEUES; 488 props->max_qp_wr = MAX_SEND_BUFFERS_PER_QUEUE; 489 490 /* 491 * max_cqe could be potentially much bigger. 492 * As this version of driver only support RAW QP, set it to the same 493 * value as max_qp_wr 494 */ 495 props->max_cqe = MAX_SEND_BUFFERS_PER_QUEUE; 496 497 props->max_mr_size = MANA_IB_MAX_MR_SIZE; 498 props->max_mr = MANA_IB_MAX_MR; 499 props->max_send_sge = MAX_TX_WQE_SGL_ENTRIES; 500 props->max_recv_sge = MAX_RX_WQE_SGL_ENTRIES; 501 502 return 0; 503 } 504 505 int mana_ib_query_port(struct ib_device *ibdev, u32 port, 506 struct ib_port_attr *props) 507 { 508 /* This version doesn't return port properties */ 509 return 0; 510 } 511 512 int mana_ib_query_gid(struct ib_device *ibdev, u32 port, int index, 513 union ib_gid *gid) 514 { 515 /* This version doesn't return GID properties */ 516 return 0; 517 } 518 519 void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext) 520 { 521 } 522