1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <linux/module.h> 5 #include <linux/pci.h> 6 #include <linux/utsname.h> 7 #include <linux/version.h> 8 9 #include <net/mana/mana.h> 10 11 static u32 mana_gd_r32(struct gdma_context *g, u64 offset) 12 { 13 return readl(g->bar0_va + offset); 14 } 15 16 static u64 mana_gd_r64(struct gdma_context *g, u64 offset) 17 { 18 return readq(g->bar0_va + offset); 19 } 20 21 static void mana_gd_init_pf_regs(struct pci_dev *pdev) 22 { 23 struct gdma_context *gc = pci_get_drvdata(pdev); 24 void __iomem *sriov_base_va; 25 u64 sriov_base_off; 26 27 gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; 28 gc->db_page_base = gc->bar0_va + 29 mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); 30 31 sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); 32 33 sriov_base_va = gc->bar0_va + sriov_base_off; 34 gc->shm_base = sriov_base_va + 35 mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF); 36 } 37 38 static void mana_gd_init_vf_regs(struct pci_dev *pdev) 39 { 40 struct gdma_context *gc = pci_get_drvdata(pdev); 41 42 gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; 43 44 gc->db_page_base = gc->bar0_va + 45 mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); 46 47 gc->phys_db_page_base = gc->bar0_pa + 48 mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); 49 50 gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET); 51 } 52 53 static void mana_gd_init_registers(struct pci_dev *pdev) 54 { 55 struct gdma_context *gc = pci_get_drvdata(pdev); 56 57 if (gc->is_pf) 58 mana_gd_init_pf_regs(pdev); 59 else 60 mana_gd_init_vf_regs(pdev); 61 } 62 63 static int mana_gd_query_max_resources(struct pci_dev *pdev) 64 { 65 struct gdma_context *gc = pci_get_drvdata(pdev); 66 struct gdma_query_max_resources_resp resp = {}; 67 struct gdma_general_req req = {}; 68 int err; 69 70 mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, 71 sizeof(req), sizeof(resp)); 72 73 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 74 if (err || resp.hdr.status) { 75 dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n", 76 err, resp.hdr.status); 77 return err ? err : -EPROTO; 78 } 79 80 if (gc->num_msix_usable > resp.max_msix) 81 gc->num_msix_usable = resp.max_msix; 82 83 if (gc->num_msix_usable <= 1) 84 return -ENOSPC; 85 86 gc->max_num_queues = num_online_cpus(); 87 if (gc->max_num_queues > MANA_MAX_NUM_QUEUES) 88 gc->max_num_queues = MANA_MAX_NUM_QUEUES; 89 90 if (gc->max_num_queues > resp.max_eq) 91 gc->max_num_queues = resp.max_eq; 92 93 if (gc->max_num_queues > resp.max_cq) 94 gc->max_num_queues = resp.max_cq; 95 96 if (gc->max_num_queues > resp.max_sq) 97 gc->max_num_queues = resp.max_sq; 98 99 if (gc->max_num_queues > resp.max_rq) 100 gc->max_num_queues = resp.max_rq; 101 102 /* The Hardware Channel (HWC) used 1 MSI-X */ 103 if (gc->max_num_queues > gc->num_msix_usable - 1) 104 gc->max_num_queues = gc->num_msix_usable - 1; 105 106 return 0; 107 } 108 109 static int mana_gd_detect_devices(struct pci_dev *pdev) 110 { 111 struct gdma_context *gc = pci_get_drvdata(pdev); 112 struct gdma_list_devices_resp resp = {}; 113 struct gdma_general_req req = {}; 114 struct gdma_dev_id dev; 115 u32 i, max_num_devs; 116 u16 dev_type; 117 int err; 118 119 mana_gd_init_req_hdr(&req.hdr, GDMA_LIST_DEVICES, sizeof(req), 120 sizeof(resp)); 121 122 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 123 if (err || resp.hdr.status) { 124 dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err, 125 resp.hdr.status); 126 return err ? err : -EPROTO; 127 } 128 129 max_num_devs = min_t(u32, MAX_NUM_GDMA_DEVICES, resp.num_of_devs); 130 131 for (i = 0; i < max_num_devs; i++) { 132 dev = resp.devs[i]; 133 dev_type = dev.type; 134 135 /* HWC is already detected in mana_hwc_create_channel(). */ 136 if (dev_type == GDMA_DEVICE_HWC) 137 continue; 138 139 if (dev_type == GDMA_DEVICE_MANA) { 140 gc->mana.gdma_context = gc; 141 gc->mana.dev_id = dev; 142 } 143 } 144 145 return gc->mana.dev_id.type == 0 ? -ENODEV : 0; 146 } 147 148 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, 149 u32 resp_len, void *resp) 150 { 151 struct hw_channel_context *hwc = gc->hwc.driver_data; 152 153 return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); 154 } 155 EXPORT_SYMBOL_NS(mana_gd_send_request, NET_MANA); 156 157 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, 158 struct gdma_mem_info *gmi) 159 { 160 dma_addr_t dma_handle; 161 void *buf; 162 163 if (length < PAGE_SIZE || !is_power_of_2(length)) 164 return -EINVAL; 165 166 gmi->dev = gc->dev; 167 buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, GFP_KERNEL); 168 if (!buf) 169 return -ENOMEM; 170 171 gmi->dma_handle = dma_handle; 172 gmi->virt_addr = buf; 173 gmi->length = length; 174 175 return 0; 176 } 177 178 void mana_gd_free_memory(struct gdma_mem_info *gmi) 179 { 180 dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr, 181 gmi->dma_handle); 182 } 183 184 static int mana_gd_create_hw_eq(struct gdma_context *gc, 185 struct gdma_queue *queue) 186 { 187 struct gdma_create_queue_resp resp = {}; 188 struct gdma_create_queue_req req = {}; 189 int err; 190 191 if (queue->type != GDMA_EQ) 192 return -EINVAL; 193 194 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_QUEUE, 195 sizeof(req), sizeof(resp)); 196 197 req.hdr.dev_id = queue->gdma_dev->dev_id; 198 req.type = queue->type; 199 req.pdid = queue->gdma_dev->pdid; 200 req.doolbell_id = queue->gdma_dev->doorbell; 201 req.gdma_region = queue->mem_info.dma_region_handle; 202 req.queue_size = queue->queue_size; 203 req.log2_throttle_limit = queue->eq.log2_throttle_limit; 204 req.eq_pci_msix_index = queue->eq.msix_index; 205 206 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 207 if (err || resp.hdr.status) { 208 dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err, 209 resp.hdr.status); 210 return err ? err : -EPROTO; 211 } 212 213 queue->id = resp.queue_index; 214 queue->eq.disable_needed = true; 215 queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; 216 return 0; 217 } 218 219 static int mana_gd_disable_queue(struct gdma_queue *queue) 220 { 221 struct gdma_context *gc = queue->gdma_dev->gdma_context; 222 struct gdma_disable_queue_req req = {}; 223 struct gdma_general_resp resp = {}; 224 int err; 225 226 WARN_ON(queue->type != GDMA_EQ); 227 228 mana_gd_init_req_hdr(&req.hdr, GDMA_DISABLE_QUEUE, 229 sizeof(req), sizeof(resp)); 230 231 req.hdr.dev_id = queue->gdma_dev->dev_id; 232 req.type = queue->type; 233 req.queue_index = queue->id; 234 req.alloc_res_id_on_creation = 1; 235 236 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 237 if (err || resp.hdr.status) { 238 dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err, 239 resp.hdr.status); 240 return err ? err : -EPROTO; 241 } 242 243 return 0; 244 } 245 246 #define DOORBELL_OFFSET_SQ 0x0 247 #define DOORBELL_OFFSET_RQ 0x400 248 #define DOORBELL_OFFSET_CQ 0x800 249 #define DOORBELL_OFFSET_EQ 0xFF8 250 251 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, 252 enum gdma_queue_type q_type, u32 qid, 253 u32 tail_ptr, u8 num_req) 254 { 255 void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; 256 union gdma_doorbell_entry e = {}; 257 258 switch (q_type) { 259 case GDMA_EQ: 260 e.eq.id = qid; 261 e.eq.tail_ptr = tail_ptr; 262 e.eq.arm = num_req; 263 264 addr += DOORBELL_OFFSET_EQ; 265 break; 266 267 case GDMA_CQ: 268 e.cq.id = qid; 269 e.cq.tail_ptr = tail_ptr; 270 e.cq.arm = num_req; 271 272 addr += DOORBELL_OFFSET_CQ; 273 break; 274 275 case GDMA_RQ: 276 e.rq.id = qid; 277 e.rq.tail_ptr = tail_ptr; 278 e.rq.wqe_cnt = num_req; 279 280 addr += DOORBELL_OFFSET_RQ; 281 break; 282 283 case GDMA_SQ: 284 e.sq.id = qid; 285 e.sq.tail_ptr = tail_ptr; 286 287 addr += DOORBELL_OFFSET_SQ; 288 break; 289 290 default: 291 WARN_ON(1); 292 return; 293 } 294 295 /* Ensure all writes are done before ring doorbell */ 296 wmb(); 297 298 writeq(e.as_uint64, addr); 299 } 300 301 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) 302 { 303 /* Hardware Spec specifies that software client should set 0 for 304 * wqe_cnt for Receive Queues. This value is not used in Send Queues. 305 */ 306 mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, 307 queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); 308 } 309 310 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) 311 { 312 struct gdma_context *gc = cq->gdma_dev->gdma_context; 313 314 u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE; 315 316 u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); 317 318 mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, 319 head, arm_bit); 320 } 321 322 static void mana_gd_process_eqe(struct gdma_queue *eq) 323 { 324 u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); 325 struct gdma_context *gc = eq->gdma_dev->gdma_context; 326 struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr; 327 union gdma_eqe_info eqe_info; 328 enum gdma_eqe_type type; 329 struct gdma_event event; 330 struct gdma_queue *cq; 331 struct gdma_eqe *eqe; 332 u32 cq_id; 333 334 eqe = &eq_eqe_ptr[head]; 335 eqe_info.as_uint32 = eqe->eqe_info; 336 type = eqe_info.type; 337 338 switch (type) { 339 case GDMA_EQE_COMPLETION: 340 cq_id = eqe->details[0] & 0xFFFFFF; 341 if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) 342 break; 343 344 cq = gc->cq_table[cq_id]; 345 if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id)) 346 break; 347 348 if (cq->cq.callback) 349 cq->cq.callback(cq->cq.context, cq); 350 351 break; 352 353 case GDMA_EQE_TEST_EVENT: 354 gc->test_event_eq_id = eq->id; 355 complete(&gc->eq_test_event); 356 break; 357 358 case GDMA_EQE_HWC_INIT_EQ_ID_DB: 359 case GDMA_EQE_HWC_INIT_DATA: 360 case GDMA_EQE_HWC_INIT_DONE: 361 if (!eq->eq.callback) 362 break; 363 364 event.type = type; 365 memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE); 366 eq->eq.callback(eq->eq.context, eq, &event); 367 break; 368 369 default: 370 break; 371 } 372 } 373 374 static void mana_gd_process_eq_events(void *arg) 375 { 376 u32 owner_bits, new_bits, old_bits; 377 union gdma_eqe_info eqe_info; 378 struct gdma_eqe *eq_eqe_ptr; 379 struct gdma_queue *eq = arg; 380 struct gdma_context *gc; 381 struct gdma_eqe *eqe; 382 u32 head, num_eqe; 383 int i; 384 385 gc = eq->gdma_dev->gdma_context; 386 387 num_eqe = eq->queue_size / GDMA_EQE_SIZE; 388 eq_eqe_ptr = eq->queue_mem_ptr; 389 390 /* Process up to 5 EQEs at a time, and update the HW head. */ 391 for (i = 0; i < 5; i++) { 392 eqe = &eq_eqe_ptr[eq->head % num_eqe]; 393 eqe_info.as_uint32 = eqe->eqe_info; 394 owner_bits = eqe_info.owner_bits; 395 396 old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; 397 /* No more entries */ 398 if (owner_bits == old_bits) 399 break; 400 401 new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; 402 if (owner_bits != new_bits) { 403 dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id); 404 break; 405 } 406 407 /* Per GDMA spec, rmb is necessary after checking owner_bits, before 408 * reading eqe. 409 */ 410 rmb(); 411 412 mana_gd_process_eqe(eq); 413 414 eq->head++; 415 } 416 417 head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); 418 419 mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, 420 head, SET_ARM_BIT); 421 } 422 423 static int mana_gd_register_irq(struct gdma_queue *queue, 424 const struct gdma_queue_spec *spec) 425 { 426 struct gdma_dev *gd = queue->gdma_dev; 427 struct gdma_irq_context *gic; 428 struct gdma_context *gc; 429 struct gdma_resource *r; 430 unsigned int msi_index; 431 unsigned long flags; 432 struct device *dev; 433 int err = 0; 434 435 gc = gd->gdma_context; 436 r = &gc->msix_resource; 437 dev = gc->dev; 438 439 spin_lock_irqsave(&r->lock, flags); 440 441 msi_index = find_first_zero_bit(r->map, r->size); 442 if (msi_index >= r->size || msi_index >= gc->num_msix_usable) { 443 err = -ENOSPC; 444 } else { 445 bitmap_set(r->map, msi_index, 1); 446 queue->eq.msix_index = msi_index; 447 } 448 449 spin_unlock_irqrestore(&r->lock, flags); 450 451 if (err) { 452 dev_err(dev, "Register IRQ err:%d, msi:%u rsize:%u, nMSI:%u", 453 err, msi_index, r->size, gc->num_msix_usable); 454 455 return err; 456 } 457 458 gic = &gc->irq_contexts[msi_index]; 459 460 WARN_ON(gic->handler || gic->arg); 461 462 gic->arg = queue; 463 464 gic->handler = mana_gd_process_eq_events; 465 466 return 0; 467 } 468 469 static void mana_gd_deregiser_irq(struct gdma_queue *queue) 470 { 471 struct gdma_dev *gd = queue->gdma_dev; 472 struct gdma_irq_context *gic; 473 struct gdma_context *gc; 474 struct gdma_resource *r; 475 unsigned int msix_index; 476 unsigned long flags; 477 478 gc = gd->gdma_context; 479 r = &gc->msix_resource; 480 481 /* At most num_online_cpus() + 1 interrupts are used. */ 482 msix_index = queue->eq.msix_index; 483 if (WARN_ON(msix_index >= gc->num_msix_usable)) 484 return; 485 486 gic = &gc->irq_contexts[msix_index]; 487 gic->handler = NULL; 488 gic->arg = NULL; 489 490 spin_lock_irqsave(&r->lock, flags); 491 bitmap_clear(r->map, msix_index, 1); 492 spin_unlock_irqrestore(&r->lock, flags); 493 494 queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; 495 } 496 497 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) 498 { 499 struct gdma_generate_test_event_req req = {}; 500 struct gdma_general_resp resp = {}; 501 struct device *dev = gc->dev; 502 int err; 503 504 mutex_lock(&gc->eq_test_event_mutex); 505 506 init_completion(&gc->eq_test_event); 507 gc->test_event_eq_id = INVALID_QUEUE_ID; 508 509 mana_gd_init_req_hdr(&req.hdr, GDMA_GENERATE_TEST_EQE, 510 sizeof(req), sizeof(resp)); 511 512 req.hdr.dev_id = eq->gdma_dev->dev_id; 513 req.queue_index = eq->id; 514 515 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 516 if (err) { 517 dev_err(dev, "test_eq failed: %d\n", err); 518 goto out; 519 } 520 521 err = -EPROTO; 522 523 if (resp.hdr.status) { 524 dev_err(dev, "test_eq failed: 0x%x\n", resp.hdr.status); 525 goto out; 526 } 527 528 if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) { 529 dev_err(dev, "test_eq timed out on queue %d\n", eq->id); 530 goto out; 531 } 532 533 if (eq->id != gc->test_event_eq_id) { 534 dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n", 535 gc->test_event_eq_id, eq->id); 536 goto out; 537 } 538 539 err = 0; 540 out: 541 mutex_unlock(&gc->eq_test_event_mutex); 542 return err; 543 } 544 545 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets, 546 struct gdma_queue *queue) 547 { 548 int err; 549 550 if (flush_evenets) { 551 err = mana_gd_test_eq(gc, queue); 552 if (err) 553 dev_warn(gc->dev, "Failed to flush EQ: %d\n", err); 554 } 555 556 mana_gd_deregiser_irq(queue); 557 558 if (queue->eq.disable_needed) 559 mana_gd_disable_queue(queue); 560 } 561 562 static int mana_gd_create_eq(struct gdma_dev *gd, 563 const struct gdma_queue_spec *spec, 564 bool create_hwq, struct gdma_queue *queue) 565 { 566 struct gdma_context *gc = gd->gdma_context; 567 struct device *dev = gc->dev; 568 u32 log2_num_entries; 569 int err; 570 571 queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; 572 573 log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); 574 575 if (spec->eq.log2_throttle_limit > log2_num_entries) { 576 dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n", 577 spec->eq.log2_throttle_limit, log2_num_entries); 578 return -EINVAL; 579 } 580 581 err = mana_gd_register_irq(queue, spec); 582 if (err) { 583 dev_err(dev, "Failed to register irq: %d\n", err); 584 return err; 585 } 586 587 queue->eq.callback = spec->eq.callback; 588 queue->eq.context = spec->eq.context; 589 queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); 590 queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1; 591 592 if (create_hwq) { 593 err = mana_gd_create_hw_eq(gc, queue); 594 if (err) 595 goto out; 596 597 err = mana_gd_test_eq(gc, queue); 598 if (err) 599 goto out; 600 } 601 602 return 0; 603 out: 604 dev_err(dev, "Failed to create EQ: %d\n", err); 605 mana_gd_destroy_eq(gc, false, queue); 606 return err; 607 } 608 609 static void mana_gd_create_cq(const struct gdma_queue_spec *spec, 610 struct gdma_queue *queue) 611 { 612 u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE); 613 614 queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); 615 queue->cq.parent = spec->cq.parent_eq; 616 queue->cq.context = spec->cq.context; 617 queue->cq.callback = spec->cq.callback; 618 } 619 620 static void mana_gd_destroy_cq(struct gdma_context *gc, 621 struct gdma_queue *queue) 622 { 623 u32 id = queue->id; 624 625 if (id >= gc->max_num_cqs) 626 return; 627 628 if (!gc->cq_table[id]) 629 return; 630 631 gc->cq_table[id] = NULL; 632 } 633 634 int mana_gd_create_hwc_queue(struct gdma_dev *gd, 635 const struct gdma_queue_spec *spec, 636 struct gdma_queue **queue_ptr) 637 { 638 struct gdma_context *gc = gd->gdma_context; 639 struct gdma_mem_info *gmi; 640 struct gdma_queue *queue; 641 int err; 642 643 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 644 if (!queue) 645 return -ENOMEM; 646 647 gmi = &queue->mem_info; 648 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); 649 if (err) 650 goto free_q; 651 652 queue->head = 0; 653 queue->tail = 0; 654 queue->queue_mem_ptr = gmi->virt_addr; 655 queue->queue_size = spec->queue_size; 656 queue->monitor_avl_buf = spec->monitor_avl_buf; 657 queue->type = spec->type; 658 queue->gdma_dev = gd; 659 660 if (spec->type == GDMA_EQ) 661 err = mana_gd_create_eq(gd, spec, false, queue); 662 else if (spec->type == GDMA_CQ) 663 mana_gd_create_cq(spec, queue); 664 665 if (err) 666 goto out; 667 668 *queue_ptr = queue; 669 return 0; 670 out: 671 mana_gd_free_memory(gmi); 672 free_q: 673 kfree(queue); 674 return err; 675 } 676 677 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle) 678 { 679 struct gdma_destroy_dma_region_req req = {}; 680 struct gdma_general_resp resp = {}; 681 int err; 682 683 if (dma_region_handle == GDMA_INVALID_DMA_REGION) 684 return 0; 685 686 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_DMA_REGION, sizeof(req), 687 sizeof(resp)); 688 req.dma_region_handle = dma_region_handle; 689 690 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 691 if (err || resp.hdr.status) { 692 dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n", 693 err, resp.hdr.status); 694 return -EPROTO; 695 } 696 697 return 0; 698 } 699 EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, NET_MANA); 700 701 static int mana_gd_create_dma_region(struct gdma_dev *gd, 702 struct gdma_mem_info *gmi) 703 { 704 unsigned int num_page = gmi->length / PAGE_SIZE; 705 struct gdma_create_dma_region_req *req = NULL; 706 struct gdma_create_dma_region_resp resp = {}; 707 struct gdma_context *gc = gd->gdma_context; 708 struct hw_channel_context *hwc; 709 u32 length = gmi->length; 710 size_t req_msg_size; 711 int err; 712 int i; 713 714 if (length < PAGE_SIZE || !is_power_of_2(length)) 715 return -EINVAL; 716 717 if (offset_in_page(gmi->virt_addr) != 0) 718 return -EINVAL; 719 720 hwc = gc->hwc.driver_data; 721 req_msg_size = struct_size(req, page_addr_list, num_page); 722 if (req_msg_size > hwc->max_req_msg_size) 723 return -EINVAL; 724 725 req = kzalloc(req_msg_size, GFP_KERNEL); 726 if (!req) 727 return -ENOMEM; 728 729 mana_gd_init_req_hdr(&req->hdr, GDMA_CREATE_DMA_REGION, 730 req_msg_size, sizeof(resp)); 731 req->length = length; 732 req->offset_in_page = 0; 733 req->gdma_page_type = GDMA_PAGE_TYPE_4K; 734 req->page_count = num_page; 735 req->page_addr_list_len = num_page; 736 737 for (i = 0; i < num_page; i++) 738 req->page_addr_list[i] = gmi->dma_handle + i * PAGE_SIZE; 739 740 err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp); 741 if (err) 742 goto out; 743 744 if (resp.hdr.status || 745 resp.dma_region_handle == GDMA_INVALID_DMA_REGION) { 746 dev_err(gc->dev, "Failed to create DMA region: 0x%x\n", 747 resp.hdr.status); 748 err = -EPROTO; 749 goto out; 750 } 751 752 gmi->dma_region_handle = resp.dma_region_handle; 753 out: 754 kfree(req); 755 return err; 756 } 757 758 int mana_gd_create_mana_eq(struct gdma_dev *gd, 759 const struct gdma_queue_spec *spec, 760 struct gdma_queue **queue_ptr) 761 { 762 struct gdma_context *gc = gd->gdma_context; 763 struct gdma_mem_info *gmi; 764 struct gdma_queue *queue; 765 int err; 766 767 if (spec->type != GDMA_EQ) 768 return -EINVAL; 769 770 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 771 if (!queue) 772 return -ENOMEM; 773 774 gmi = &queue->mem_info; 775 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); 776 if (err) 777 goto free_q; 778 779 err = mana_gd_create_dma_region(gd, gmi); 780 if (err) 781 goto out; 782 783 queue->head = 0; 784 queue->tail = 0; 785 queue->queue_mem_ptr = gmi->virt_addr; 786 queue->queue_size = spec->queue_size; 787 queue->monitor_avl_buf = spec->monitor_avl_buf; 788 queue->type = spec->type; 789 queue->gdma_dev = gd; 790 791 err = mana_gd_create_eq(gd, spec, true, queue); 792 if (err) 793 goto out; 794 795 *queue_ptr = queue; 796 return 0; 797 out: 798 mana_gd_free_memory(gmi); 799 free_q: 800 kfree(queue); 801 return err; 802 } 803 804 int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, 805 const struct gdma_queue_spec *spec, 806 struct gdma_queue **queue_ptr) 807 { 808 struct gdma_context *gc = gd->gdma_context; 809 struct gdma_mem_info *gmi; 810 struct gdma_queue *queue; 811 int err; 812 813 if (spec->type != GDMA_CQ && spec->type != GDMA_SQ && 814 spec->type != GDMA_RQ) 815 return -EINVAL; 816 817 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 818 if (!queue) 819 return -ENOMEM; 820 821 gmi = &queue->mem_info; 822 err = mana_gd_alloc_memory(gc, spec->queue_size, gmi); 823 if (err) 824 goto free_q; 825 826 err = mana_gd_create_dma_region(gd, gmi); 827 if (err) 828 goto out; 829 830 queue->head = 0; 831 queue->tail = 0; 832 queue->queue_mem_ptr = gmi->virt_addr; 833 queue->queue_size = spec->queue_size; 834 queue->monitor_avl_buf = spec->monitor_avl_buf; 835 queue->type = spec->type; 836 queue->gdma_dev = gd; 837 838 if (spec->type == GDMA_CQ) 839 mana_gd_create_cq(spec, queue); 840 841 *queue_ptr = queue; 842 return 0; 843 out: 844 mana_gd_free_memory(gmi); 845 free_q: 846 kfree(queue); 847 return err; 848 } 849 850 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) 851 { 852 struct gdma_mem_info *gmi = &queue->mem_info; 853 854 switch (queue->type) { 855 case GDMA_EQ: 856 mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue); 857 break; 858 859 case GDMA_CQ: 860 mana_gd_destroy_cq(gc, queue); 861 break; 862 863 case GDMA_RQ: 864 break; 865 866 case GDMA_SQ: 867 break; 868 869 default: 870 dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n", 871 queue->type); 872 return; 873 } 874 875 mana_gd_destroy_dma_region(gc, gmi->dma_region_handle); 876 mana_gd_free_memory(gmi); 877 kfree(queue); 878 } 879 880 int mana_gd_verify_vf_version(struct pci_dev *pdev) 881 { 882 struct gdma_context *gc = pci_get_drvdata(pdev); 883 struct gdma_verify_ver_resp resp = {}; 884 struct gdma_verify_ver_req req = {}; 885 int err; 886 887 mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION, 888 sizeof(req), sizeof(resp)); 889 890 req.protocol_ver_min = GDMA_PROTOCOL_FIRST; 891 req.protocol_ver_max = GDMA_PROTOCOL_LAST; 892 893 req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; 894 req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; 895 req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; 896 req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; 897 898 req.drv_ver = 0; /* Unused*/ 899 req.os_type = 0x10; /* Linux */ 900 req.os_ver_major = LINUX_VERSION_MAJOR; 901 req.os_ver_minor = LINUX_VERSION_PATCHLEVEL; 902 req.os_ver_build = LINUX_VERSION_SUBLEVEL; 903 strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1)); 904 strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2)); 905 strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3)); 906 907 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 908 if (err || resp.hdr.status) { 909 dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n", 910 err, resp.hdr.status); 911 return err ? err : -EPROTO; 912 } 913 914 return 0; 915 } 916 917 int mana_gd_register_device(struct gdma_dev *gd) 918 { 919 struct gdma_context *gc = gd->gdma_context; 920 struct gdma_register_device_resp resp = {}; 921 struct gdma_general_req req = {}; 922 int err; 923 924 gd->pdid = INVALID_PDID; 925 gd->doorbell = INVALID_DOORBELL; 926 gd->gpa_mkey = INVALID_MEM_KEY; 927 928 mana_gd_init_req_hdr(&req.hdr, GDMA_REGISTER_DEVICE, sizeof(req), 929 sizeof(resp)); 930 931 req.hdr.dev_id = gd->dev_id; 932 933 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 934 if (err || resp.hdr.status) { 935 dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n", 936 err, resp.hdr.status); 937 return err ? err : -EPROTO; 938 } 939 940 gd->pdid = resp.pdid; 941 gd->gpa_mkey = resp.gpa_mkey; 942 gd->doorbell = resp.db_id; 943 944 return 0; 945 } 946 947 int mana_gd_deregister_device(struct gdma_dev *gd) 948 { 949 struct gdma_context *gc = gd->gdma_context; 950 struct gdma_general_resp resp = {}; 951 struct gdma_general_req req = {}; 952 int err; 953 954 if (gd->pdid == INVALID_PDID) 955 return -EINVAL; 956 957 mana_gd_init_req_hdr(&req.hdr, GDMA_DEREGISTER_DEVICE, sizeof(req), 958 sizeof(resp)); 959 960 req.hdr.dev_id = gd->dev_id; 961 962 err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); 963 if (err || resp.hdr.status) { 964 dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n", 965 err, resp.hdr.status); 966 if (!err) 967 err = -EPROTO; 968 } 969 970 gd->pdid = INVALID_PDID; 971 gd->doorbell = INVALID_DOORBELL; 972 gd->gpa_mkey = INVALID_MEM_KEY; 973 974 return err; 975 } 976 977 u32 mana_gd_wq_avail_space(struct gdma_queue *wq) 978 { 979 u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE; 980 u32 wq_size = wq->queue_size; 981 982 WARN_ON_ONCE(used_space > wq_size); 983 984 return wq_size - used_space; 985 } 986 987 u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset) 988 { 989 u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); 990 991 WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size); 992 993 return wq->queue_mem_ptr + offset; 994 } 995 996 static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, 997 enum gdma_queue_type q_type, 998 u32 client_oob_size, u32 sgl_data_size, 999 u8 *wqe_ptr) 1000 { 1001 bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL); 1002 bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0); 1003 struct gdma_wqe *header = (struct gdma_wqe *)wqe_ptr; 1004 u8 *ptr; 1005 1006 memset(header, 0, sizeof(struct gdma_wqe)); 1007 header->num_sge = wqe_req->num_sge; 1008 header->inline_oob_size_div4 = client_oob_size / sizeof(u32); 1009 1010 if (oob_in_sgl) { 1011 WARN_ON_ONCE(!pad_data || wqe_req->num_sge < 2); 1012 1013 header->client_oob_in_sgl = 1; 1014 1015 if (pad_data) 1016 header->last_vbytes = wqe_req->sgl[0].size; 1017 } 1018 1019 if (q_type == GDMA_SQ) 1020 header->client_data_unit = wqe_req->client_data_unit; 1021 1022 /* The size of gdma_wqe + client_oob_size must be less than or equal 1023 * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond 1024 * the queue memory buffer boundary. 1025 */ 1026 ptr = wqe_ptr + sizeof(header); 1027 1028 if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) { 1029 memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size); 1030 1031 if (client_oob_size > wqe_req->inline_oob_size) 1032 memset(ptr + wqe_req->inline_oob_size, 0, 1033 client_oob_size - wqe_req->inline_oob_size); 1034 } 1035 1036 return sizeof(header) + client_oob_size; 1037 } 1038 1039 static void mana_gd_write_sgl(struct gdma_queue *wq, u8 *wqe_ptr, 1040 const struct gdma_wqe_request *wqe_req) 1041 { 1042 u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge; 1043 const u8 *address = (u8 *)wqe_req->sgl; 1044 u8 *base_ptr, *end_ptr; 1045 u32 size_to_end; 1046 1047 base_ptr = wq->queue_mem_ptr; 1048 end_ptr = base_ptr + wq->queue_size; 1049 size_to_end = (u32)(end_ptr - wqe_ptr); 1050 1051 if (size_to_end < sgl_size) { 1052 memcpy(wqe_ptr, address, size_to_end); 1053 1054 wqe_ptr = base_ptr; 1055 address += size_to_end; 1056 sgl_size -= size_to_end; 1057 } 1058 1059 memcpy(wqe_ptr, address, sgl_size); 1060 } 1061 1062 int mana_gd_post_work_request(struct gdma_queue *wq, 1063 const struct gdma_wqe_request *wqe_req, 1064 struct gdma_posted_wqe_info *wqe_info) 1065 { 1066 u32 client_oob_size = wqe_req->inline_oob_size; 1067 struct gdma_context *gc; 1068 u32 sgl_data_size; 1069 u32 max_wqe_size; 1070 u32 wqe_size; 1071 u8 *wqe_ptr; 1072 1073 if (wqe_req->num_sge == 0) 1074 return -EINVAL; 1075 1076 if (wq->type == GDMA_RQ) { 1077 if (client_oob_size != 0) 1078 return -EINVAL; 1079 1080 client_oob_size = INLINE_OOB_SMALL_SIZE; 1081 1082 max_wqe_size = GDMA_MAX_RQE_SIZE; 1083 } else { 1084 if (client_oob_size != INLINE_OOB_SMALL_SIZE && 1085 client_oob_size != INLINE_OOB_LARGE_SIZE) 1086 return -EINVAL; 1087 1088 max_wqe_size = GDMA_MAX_SQE_SIZE; 1089 } 1090 1091 sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge; 1092 wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size + 1093 sgl_data_size, GDMA_WQE_BU_SIZE); 1094 if (wqe_size > max_wqe_size) 1095 return -EINVAL; 1096 1097 if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) { 1098 gc = wq->gdma_dev->gdma_context; 1099 dev_err(gc->dev, "unsuccessful flow control!\n"); 1100 return -ENOSPC; 1101 } 1102 1103 if (wqe_info) 1104 wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; 1105 1106 wqe_ptr = mana_gd_get_wqe_ptr(wq, wq->head); 1107 wqe_ptr += mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, 1108 sgl_data_size, wqe_ptr); 1109 if (wqe_ptr >= (u8 *)wq->queue_mem_ptr + wq->queue_size) 1110 wqe_ptr -= wq->queue_size; 1111 1112 mana_gd_write_sgl(wq, wqe_ptr, wqe_req); 1113 1114 wq->head += wqe_size / GDMA_WQE_BU_SIZE; 1115 1116 return 0; 1117 } 1118 1119 int mana_gd_post_and_ring(struct gdma_queue *queue, 1120 const struct gdma_wqe_request *wqe_req, 1121 struct gdma_posted_wqe_info *wqe_info) 1122 { 1123 struct gdma_context *gc = queue->gdma_dev->gdma_context; 1124 int err; 1125 1126 err = mana_gd_post_work_request(queue, wqe_req, wqe_info); 1127 if (err) 1128 return err; 1129 1130 mana_gd_wq_ring_doorbell(gc, queue); 1131 1132 return 0; 1133 } 1134 1135 static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) 1136 { 1137 unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe); 1138 struct gdma_cqe *cq_cqe = cq->queue_mem_ptr; 1139 u32 owner_bits, new_bits, old_bits; 1140 struct gdma_cqe *cqe; 1141 1142 cqe = &cq_cqe[cq->head % num_cqe]; 1143 owner_bits = cqe->cqe_info.owner_bits; 1144 1145 old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK; 1146 /* Return 0 if no more entries. */ 1147 if (owner_bits == old_bits) 1148 return 0; 1149 1150 new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK; 1151 /* Return -1 if overflow detected. */ 1152 if (WARN_ON_ONCE(owner_bits != new_bits)) 1153 return -1; 1154 1155 /* Per GDMA spec, rmb is necessary after checking owner_bits, before 1156 * reading completion info 1157 */ 1158 rmb(); 1159 1160 comp->wq_num = cqe->cqe_info.wq_num; 1161 comp->is_sq = cqe->cqe_info.is_sq; 1162 memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); 1163 1164 return 1; 1165 } 1166 1167 int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) 1168 { 1169 int cqe_idx; 1170 int ret; 1171 1172 for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) { 1173 ret = mana_gd_read_cqe(cq, &comp[cqe_idx]); 1174 1175 if (ret < 0) { 1176 cq->head -= cqe_idx; 1177 return ret; 1178 } 1179 1180 if (ret == 0) 1181 break; 1182 1183 cq->head++; 1184 } 1185 1186 return cqe_idx; 1187 } 1188 1189 static irqreturn_t mana_gd_intr(int irq, void *arg) 1190 { 1191 struct gdma_irq_context *gic = arg; 1192 1193 if (gic->handler) 1194 gic->handler(gic->arg); 1195 1196 return IRQ_HANDLED; 1197 } 1198 1199 int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r) 1200 { 1201 r->map = bitmap_zalloc(res_avail, GFP_KERNEL); 1202 if (!r->map) 1203 return -ENOMEM; 1204 1205 r->size = res_avail; 1206 spin_lock_init(&r->lock); 1207 1208 return 0; 1209 } 1210 1211 void mana_gd_free_res_map(struct gdma_resource *r) 1212 { 1213 bitmap_free(r->map); 1214 r->map = NULL; 1215 r->size = 0; 1216 } 1217 1218 static int mana_gd_setup_irqs(struct pci_dev *pdev) 1219 { 1220 unsigned int max_queues_per_port = num_online_cpus(); 1221 struct gdma_context *gc = pci_get_drvdata(pdev); 1222 struct gdma_irq_context *gic; 1223 unsigned int max_irqs, cpu; 1224 int nvec, irq; 1225 int err, i = 0, j; 1226 1227 if (max_queues_per_port > MANA_MAX_NUM_QUEUES) 1228 max_queues_per_port = MANA_MAX_NUM_QUEUES; 1229 1230 /* Need 1 interrupt for the Hardware communication Channel (HWC) */ 1231 max_irqs = max_queues_per_port + 1; 1232 1233 nvec = pci_alloc_irq_vectors(pdev, 2, max_irqs, PCI_IRQ_MSIX); 1234 if (nvec < 0) 1235 return nvec; 1236 1237 gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context), 1238 GFP_KERNEL); 1239 if (!gc->irq_contexts) { 1240 err = -ENOMEM; 1241 goto free_irq_vector; 1242 } 1243 1244 for (i = 0; i < nvec; i++) { 1245 gic = &gc->irq_contexts[i]; 1246 gic->handler = NULL; 1247 gic->arg = NULL; 1248 1249 if (!i) 1250 snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", 1251 pci_name(pdev)); 1252 else 1253 snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_q%d@pci:%s", 1254 i - 1, pci_name(pdev)); 1255 1256 irq = pci_irq_vector(pdev, i); 1257 if (irq < 0) { 1258 err = irq; 1259 goto free_irq; 1260 } 1261 1262 err = request_irq(irq, mana_gd_intr, 0, gic->name, gic); 1263 if (err) 1264 goto free_irq; 1265 1266 cpu = cpumask_local_spread(i, gc->numa_node); 1267 irq_set_affinity_and_hint(irq, cpumask_of(cpu)); 1268 } 1269 1270 err = mana_gd_alloc_res_map(nvec, &gc->msix_resource); 1271 if (err) 1272 goto free_irq; 1273 1274 gc->max_num_msix = nvec; 1275 gc->num_msix_usable = nvec; 1276 1277 return 0; 1278 1279 free_irq: 1280 for (j = i - 1; j >= 0; j--) { 1281 irq = pci_irq_vector(pdev, j); 1282 gic = &gc->irq_contexts[j]; 1283 1284 irq_update_affinity_hint(irq, NULL); 1285 free_irq(irq, gic); 1286 } 1287 1288 kfree(gc->irq_contexts); 1289 gc->irq_contexts = NULL; 1290 free_irq_vector: 1291 pci_free_irq_vectors(pdev); 1292 return err; 1293 } 1294 1295 static void mana_gd_remove_irqs(struct pci_dev *pdev) 1296 { 1297 struct gdma_context *gc = pci_get_drvdata(pdev); 1298 struct gdma_irq_context *gic; 1299 int irq, i; 1300 1301 if (gc->max_num_msix < 1) 1302 return; 1303 1304 mana_gd_free_res_map(&gc->msix_resource); 1305 1306 for (i = 0; i < gc->max_num_msix; i++) { 1307 irq = pci_irq_vector(pdev, i); 1308 if (irq < 0) 1309 continue; 1310 1311 gic = &gc->irq_contexts[i]; 1312 1313 /* Need to clear the hint before free_irq */ 1314 irq_update_affinity_hint(irq, NULL); 1315 free_irq(irq, gic); 1316 } 1317 1318 pci_free_irq_vectors(pdev); 1319 1320 gc->max_num_msix = 0; 1321 gc->num_msix_usable = 0; 1322 kfree(gc->irq_contexts); 1323 gc->irq_contexts = NULL; 1324 } 1325 1326 static int mana_gd_setup(struct pci_dev *pdev) 1327 { 1328 struct gdma_context *gc = pci_get_drvdata(pdev); 1329 int err; 1330 1331 mana_gd_init_registers(pdev); 1332 mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); 1333 1334 err = mana_gd_setup_irqs(pdev); 1335 if (err) 1336 return err; 1337 1338 err = mana_hwc_create_channel(gc); 1339 if (err) 1340 goto remove_irq; 1341 1342 err = mana_gd_verify_vf_version(pdev); 1343 if (err) 1344 goto destroy_hwc; 1345 1346 err = mana_gd_query_max_resources(pdev); 1347 if (err) 1348 goto destroy_hwc; 1349 1350 err = mana_gd_detect_devices(pdev); 1351 if (err) 1352 goto destroy_hwc; 1353 1354 return 0; 1355 1356 destroy_hwc: 1357 mana_hwc_destroy_channel(gc); 1358 remove_irq: 1359 mana_gd_remove_irqs(pdev); 1360 return err; 1361 } 1362 1363 static void mana_gd_cleanup(struct pci_dev *pdev) 1364 { 1365 struct gdma_context *gc = pci_get_drvdata(pdev); 1366 1367 mana_hwc_destroy_channel(gc); 1368 1369 mana_gd_remove_irqs(pdev); 1370 } 1371 1372 static bool mana_is_pf(unsigned short dev_id) 1373 { 1374 return dev_id == MANA_PF_DEVICE_ID; 1375 } 1376 1377 static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1378 { 1379 struct gdma_context *gc; 1380 void __iomem *bar0_va; 1381 int bar = 0; 1382 int err; 1383 1384 /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ 1385 BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); 1386 1387 err = pci_enable_device(pdev); 1388 if (err) 1389 return -ENXIO; 1390 1391 pci_set_master(pdev); 1392 1393 err = pci_request_regions(pdev, "mana"); 1394 if (err) 1395 goto disable_dev; 1396 1397 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1398 if (err) 1399 goto release_region; 1400 1401 err = dma_set_max_seg_size(&pdev->dev, UINT_MAX); 1402 if (err) { 1403 dev_err(&pdev->dev, "Failed to set dma device segment size\n"); 1404 goto release_region; 1405 } 1406 1407 err = -ENOMEM; 1408 gc = vzalloc(sizeof(*gc)); 1409 if (!gc) 1410 goto release_region; 1411 1412 mutex_init(&gc->eq_test_event_mutex); 1413 pci_set_drvdata(pdev, gc); 1414 gc->bar0_pa = pci_resource_start(pdev, 0); 1415 1416 bar0_va = pci_iomap(pdev, bar, 0); 1417 if (!bar0_va) 1418 goto free_gc; 1419 1420 gc->numa_node = dev_to_node(&pdev->dev); 1421 gc->is_pf = mana_is_pf(pdev->device); 1422 gc->bar0_va = bar0_va; 1423 gc->dev = &pdev->dev; 1424 1425 err = mana_gd_setup(pdev); 1426 if (err) 1427 goto unmap_bar; 1428 1429 err = mana_probe(&gc->mana, false); 1430 if (err) 1431 goto cleanup_gd; 1432 1433 return 0; 1434 1435 cleanup_gd: 1436 mana_gd_cleanup(pdev); 1437 unmap_bar: 1438 pci_iounmap(pdev, bar0_va); 1439 free_gc: 1440 pci_set_drvdata(pdev, NULL); 1441 vfree(gc); 1442 release_region: 1443 pci_release_regions(pdev); 1444 disable_dev: 1445 pci_disable_device(pdev); 1446 dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); 1447 return err; 1448 } 1449 1450 static void mana_gd_remove(struct pci_dev *pdev) 1451 { 1452 struct gdma_context *gc = pci_get_drvdata(pdev); 1453 1454 mana_remove(&gc->mana, false); 1455 1456 mana_gd_cleanup(pdev); 1457 1458 pci_iounmap(pdev, gc->bar0_va); 1459 1460 vfree(gc); 1461 1462 pci_release_regions(pdev); 1463 pci_disable_device(pdev); 1464 } 1465 1466 /* The 'state' parameter is not used. */ 1467 static int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) 1468 { 1469 struct gdma_context *gc = pci_get_drvdata(pdev); 1470 1471 mana_remove(&gc->mana, true); 1472 1473 mana_gd_cleanup(pdev); 1474 1475 return 0; 1476 } 1477 1478 /* In case the NIC hardware stops working, the suspend and resume callbacks will 1479 * fail -- if this happens, it's safer to just report an error than try to undo 1480 * what has been done. 1481 */ 1482 static int mana_gd_resume(struct pci_dev *pdev) 1483 { 1484 struct gdma_context *gc = pci_get_drvdata(pdev); 1485 int err; 1486 1487 err = mana_gd_setup(pdev); 1488 if (err) 1489 return err; 1490 1491 err = mana_probe(&gc->mana, true); 1492 if (err) 1493 return err; 1494 1495 return 0; 1496 } 1497 1498 /* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ 1499 static void mana_gd_shutdown(struct pci_dev *pdev) 1500 { 1501 struct gdma_context *gc = pci_get_drvdata(pdev); 1502 1503 dev_info(&pdev->dev, "Shutdown was called\n"); 1504 1505 mana_remove(&gc->mana, true); 1506 1507 mana_gd_cleanup(pdev); 1508 1509 pci_disable_device(pdev); 1510 } 1511 1512 static const struct pci_device_id mana_id_table[] = { 1513 { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, 1514 { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, 1515 { } 1516 }; 1517 1518 static struct pci_driver mana_driver = { 1519 .name = "mana", 1520 .id_table = mana_id_table, 1521 .probe = mana_gd_probe, 1522 .remove = mana_gd_remove, 1523 .suspend = mana_gd_suspend, 1524 .resume = mana_gd_resume, 1525 .shutdown = mana_gd_shutdown, 1526 }; 1527 1528 module_pci_driver(mana_driver); 1529 1530 MODULE_DEVICE_TABLE(pci, mana_id_table); 1531 1532 MODULE_LICENSE("Dual BSD/GPL"); 1533 MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); 1534