1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/types.h> 33 #include <asm/byteorder.h> 34 #include <linux/bitops.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/etherdevice.h> 39 #include <linux/if_ether.h> 40 #include <linux/if_vlan.h> 41 #include <linux/io.h> 42 #include <linux/ip.h> 43 #include <linux/ipv6.h> 44 #include <linux/kernel.h> 45 #include <linux/list.h> 46 #include <linux/module.h> 47 #include <linux/mutex.h> 48 #include <linux/pci.h> 49 #include <linux/slab.h> 50 #include <linux/spinlock.h> 51 #include <linux/string.h> 52 #include <linux/tcp.h> 53 #include <linux/bitops.h> 54 #include <linux/qed/qed_roce_if.h> 55 #include <linux/qed/qed_roce_if.h> 56 #include "qed.h" 57 #include "qed_cxt.h" 58 #include "qed_hsi.h" 59 #include "qed_hw.h" 60 #include "qed_init_ops.h" 61 #include "qed_int.h" 62 #include "qed_ll2.h" 63 #include "qed_mcp.h" 64 #include "qed_reg_addr.h" 65 #include "qed_sp.h" 66 #include "qed_roce.h" 67 #include "qed_ll2.h" 68 69 static void qed_roce_free_real_icid(struct qed_hwfn *p_hwfn, u16 icid); 70 71 void qed_roce_async_event(struct qed_hwfn *p_hwfn, 72 u8 fw_event_code, union rdma_eqe_data *rdma_data) 73 { 74 if (fw_event_code == ROCE_ASYNC_EVENT_DESTROY_QP_DONE) { 75 u16 icid = 76 (u16)le32_to_cpu(rdma_data->rdma_destroy_qp_data.cid); 77 78 /* icid release in this async event can occur only if the icid 79 * was offloaded to the FW. In case it wasn't offloaded this is 80 * handled in qed_roce_sp_destroy_qp. 81 */ 82 qed_roce_free_real_icid(p_hwfn, icid); 83 } else { 84 struct qed_rdma_events *events = &p_hwfn->p_rdma_info->events; 85 86 events->affiliated_event(p_hwfn->p_rdma_info->events.context, 87 fw_event_code, 88 &rdma_data->async_handle); 89 } 90 } 91 92 static int qed_rdma_bmap_alloc(struct qed_hwfn *p_hwfn, 93 struct qed_bmap *bmap, u32 max_count, char *name) 94 { 95 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "max_count = %08x\n", max_count); 96 97 bmap->max_count = max_count; 98 99 bmap->bitmap = kzalloc(BITS_TO_LONGS(max_count) * sizeof(long), 100 GFP_KERNEL); 101 if (!bmap->bitmap) { 102 DP_NOTICE(p_hwfn, 103 "qed bmap alloc failed: cannot allocate memory (bitmap)\n"); 104 return -ENOMEM; 105 } 106 107 snprintf(bmap->name, QED_RDMA_MAX_BMAP_NAME, "%s", name); 108 109 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n"); 110 return 0; 111 } 112 113 static int qed_rdma_bmap_alloc_id(struct qed_hwfn *p_hwfn, 114 struct qed_bmap *bmap, u32 *id_num) 115 { 116 *id_num = find_first_zero_bit(bmap->bitmap, bmap->max_count); 117 if (*id_num >= bmap->max_count) 118 return -EINVAL; 119 120 __set_bit(*id_num, bmap->bitmap); 121 122 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "%s bitmap: allocated id %d\n", 123 bmap->name, *id_num); 124 125 return 0; 126 } 127 128 static void qed_bmap_set_id(struct qed_hwfn *p_hwfn, 129 struct qed_bmap *bmap, u32 id_num) 130 { 131 if (id_num >= bmap->max_count) 132 return; 133 134 __set_bit(id_num, bmap->bitmap); 135 } 136 137 static void qed_bmap_release_id(struct qed_hwfn *p_hwfn, 138 struct qed_bmap *bmap, u32 id_num) 139 { 140 bool b_acquired; 141 142 if (id_num >= bmap->max_count) 143 return; 144 145 b_acquired = test_and_clear_bit(id_num, bmap->bitmap); 146 if (!b_acquired) { 147 DP_NOTICE(p_hwfn, "%s bitmap: id %d already released\n", 148 bmap->name, id_num); 149 return; 150 } 151 152 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "%s bitmap: released id %d\n", 153 bmap->name, id_num); 154 } 155 156 static int qed_bmap_test_id(struct qed_hwfn *p_hwfn, 157 struct qed_bmap *bmap, u32 id_num) 158 { 159 if (id_num >= bmap->max_count) 160 return -1; 161 162 return test_bit(id_num, bmap->bitmap); 163 } 164 165 static u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id) 166 { 167 /* First sb id for RoCE is after all the l2 sb */ 168 return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id; 169 } 170 171 static int qed_rdma_alloc(struct qed_hwfn *p_hwfn, 172 struct qed_ptt *p_ptt, 173 struct qed_rdma_start_in_params *params) 174 { 175 struct qed_rdma_info *p_rdma_info; 176 u32 num_cons, num_tasks; 177 int rc = -ENOMEM; 178 179 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocating RDMA\n"); 180 181 /* Allocate a struct with current pf rdma info */ 182 p_rdma_info = kzalloc(sizeof(*p_rdma_info), GFP_KERNEL); 183 if (!p_rdma_info) { 184 DP_NOTICE(p_hwfn, 185 "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n", 186 rc); 187 return rc; 188 } 189 190 p_hwfn->p_rdma_info = p_rdma_info; 191 p_rdma_info->proto = PROTOCOLID_ROCE; 192 193 num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, 194 NULL); 195 196 p_rdma_info->num_qps = num_cons / 2; 197 198 num_tasks = qed_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE); 199 200 /* Each MR uses a single task */ 201 p_rdma_info->num_mrs = num_tasks; 202 203 /* Queue zone lines are shared between RoCE and L2 in such a way that 204 * they can be used by each without obstructing the other. 205 */ 206 p_rdma_info->queue_zone_base = (u16)RESC_START(p_hwfn, QED_L2_QUEUE); 207 p_rdma_info->max_queue_zones = (u16)RESC_NUM(p_hwfn, QED_L2_QUEUE); 208 209 /* Allocate a struct with device params and fill it */ 210 p_rdma_info->dev = kzalloc(sizeof(*p_rdma_info->dev), GFP_KERNEL); 211 if (!p_rdma_info->dev) { 212 DP_NOTICE(p_hwfn, 213 "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n", 214 rc); 215 goto free_rdma_info; 216 } 217 218 /* Allocate a struct with port params and fill it */ 219 p_rdma_info->port = kzalloc(sizeof(*p_rdma_info->port), GFP_KERNEL); 220 if (!p_rdma_info->port) { 221 DP_NOTICE(p_hwfn, 222 "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n", 223 rc); 224 goto free_rdma_dev; 225 } 226 227 /* Allocate bit map for pd's */ 228 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS, 229 "PD"); 230 if (rc) { 231 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 232 "Failed to allocate pd_map, rc = %d\n", 233 rc); 234 goto free_rdma_port; 235 } 236 237 /* Allocate DPI bitmap */ 238 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map, 239 p_hwfn->dpi_count, "DPI"); 240 if (rc) { 241 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 242 "Failed to allocate DPI bitmap, rc = %d\n", rc); 243 goto free_pd_map; 244 } 245 246 /* Allocate bitmap for cq's. The maximum number of CQs is bounded to 247 * twice the number of QPs. 248 */ 249 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map, 250 p_rdma_info->num_qps * 2, "CQ"); 251 if (rc) { 252 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 253 "Failed to allocate cq bitmap, rc = %d\n", rc); 254 goto free_dpi_map; 255 } 256 257 /* Allocate bitmap for toggle bit for cq icids 258 * We toggle the bit every time we create or resize cq for a given icid. 259 * The maximum number of CQs is bounded to twice the number of QPs. 260 */ 261 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits, 262 p_rdma_info->num_qps * 2, "Toggle"); 263 if (rc) { 264 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 265 "Failed to allocate toogle bits, rc = %d\n", rc); 266 goto free_cq_map; 267 } 268 269 /* Allocate bitmap for itids */ 270 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map, 271 p_rdma_info->num_mrs, "MR"); 272 if (rc) { 273 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 274 "Failed to allocate itids bitmaps, rc = %d\n", rc); 275 goto free_toggle_map; 276 } 277 278 /* Allocate bitmap for cids used for qps. */ 279 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons, 280 "CID"); 281 if (rc) { 282 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 283 "Failed to allocate cid bitmap, rc = %d\n", rc); 284 goto free_tid_map; 285 } 286 287 /* Allocate bitmap for cids used for responders/requesters. */ 288 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->real_cid_map, num_cons, 289 "REAL_CID"); 290 if (rc) { 291 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 292 "Failed to allocate real cid bitmap, rc = %d\n", rc); 293 goto free_cid_map; 294 } 295 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocation successful\n"); 296 return 0; 297 298 free_cid_map: 299 kfree(p_rdma_info->cid_map.bitmap); 300 free_tid_map: 301 kfree(p_rdma_info->tid_map.bitmap); 302 free_toggle_map: 303 kfree(p_rdma_info->toggle_bits.bitmap); 304 free_cq_map: 305 kfree(p_rdma_info->cq_map.bitmap); 306 free_dpi_map: 307 kfree(p_rdma_info->dpi_map.bitmap); 308 free_pd_map: 309 kfree(p_rdma_info->pd_map.bitmap); 310 free_rdma_port: 311 kfree(p_rdma_info->port); 312 free_rdma_dev: 313 kfree(p_rdma_info->dev); 314 free_rdma_info: 315 kfree(p_rdma_info); 316 317 return rc; 318 } 319 320 static void qed_rdma_bmap_free(struct qed_hwfn *p_hwfn, 321 struct qed_bmap *bmap, bool check) 322 { 323 int weight = bitmap_weight(bmap->bitmap, bmap->max_count); 324 int last_line = bmap->max_count / (64 * 8); 325 int last_item = last_line * 8 + 326 DIV_ROUND_UP(bmap->max_count % (64 * 8), 64); 327 u64 *pmap = (u64 *)bmap->bitmap; 328 int line, item, offset; 329 u8 str_last_line[200] = { 0 }; 330 331 if (!weight || !check) 332 goto end; 333 334 DP_NOTICE(p_hwfn, 335 "%s bitmap not free - size=%d, weight=%d, 512 bits per line\n", 336 bmap->name, bmap->max_count, weight); 337 338 /* print aligned non-zero lines, if any */ 339 for (item = 0, line = 0; line < last_line; line++, item += 8) 340 if (bitmap_weight((unsigned long *)&pmap[item], 64 * 8)) 341 DP_NOTICE(p_hwfn, 342 "line 0x%04x: 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n", 343 line, 344 pmap[item], 345 pmap[item + 1], 346 pmap[item + 2], 347 pmap[item + 3], 348 pmap[item + 4], 349 pmap[item + 5], 350 pmap[item + 6], pmap[item + 7]); 351 352 /* print last unaligned non-zero line, if any */ 353 if ((bmap->max_count % (64 * 8)) && 354 (bitmap_weight((unsigned long *)&pmap[item], 355 bmap->max_count - item * 64))) { 356 offset = sprintf(str_last_line, "line 0x%04x: ", line); 357 for (; item < last_item; item++) 358 offset += sprintf(str_last_line + offset, 359 "0x%016llx ", pmap[item]); 360 DP_NOTICE(p_hwfn, "%s\n", str_last_line); 361 } 362 363 end: 364 kfree(bmap->bitmap); 365 bmap->bitmap = NULL; 366 } 367 368 static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn) 369 { 370 struct qed_bmap *rcid_map = &p_hwfn->p_rdma_info->real_cid_map; 371 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 372 int wait_count = 0; 373 374 /* when destroying a_RoCE QP the control is returned to the user after 375 * the synchronous part. The asynchronous part may take a little longer. 376 * We delay for a short while if an async destroy QP is still expected. 377 * Beyond the added delay we clear the bitmap anyway. 378 */ 379 while (bitmap_weight(rcid_map->bitmap, rcid_map->max_count)) { 380 msleep(100); 381 if (wait_count++ > 20) { 382 DP_NOTICE(p_hwfn, "cid bitmap wait timed out\n"); 383 break; 384 } 385 } 386 387 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1); 388 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->pd_map, 1); 389 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 1); 390 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->cq_map, 1); 391 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->toggle_bits, 0); 392 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tid_map, 1); 393 394 kfree(p_rdma_info->port); 395 kfree(p_rdma_info->dev); 396 397 kfree(p_rdma_info); 398 } 399 400 static void qed_rdma_free(struct qed_hwfn *p_hwfn) 401 { 402 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n"); 403 404 qed_rdma_resc_free(p_hwfn); 405 } 406 407 static void qed_rdma_get_guid(struct qed_hwfn *p_hwfn, u8 *guid) 408 { 409 guid[0] = p_hwfn->hw_info.hw_mac_addr[0] ^ 2; 410 guid[1] = p_hwfn->hw_info.hw_mac_addr[1]; 411 guid[2] = p_hwfn->hw_info.hw_mac_addr[2]; 412 guid[3] = 0xff; 413 guid[4] = 0xfe; 414 guid[5] = p_hwfn->hw_info.hw_mac_addr[3]; 415 guid[6] = p_hwfn->hw_info.hw_mac_addr[4]; 416 guid[7] = p_hwfn->hw_info.hw_mac_addr[5]; 417 } 418 419 static void qed_rdma_init_events(struct qed_hwfn *p_hwfn, 420 struct qed_rdma_start_in_params *params) 421 { 422 struct qed_rdma_events *events; 423 424 events = &p_hwfn->p_rdma_info->events; 425 426 events->unaffiliated_event = params->events->unaffiliated_event; 427 events->affiliated_event = params->events->affiliated_event; 428 events->context = params->events->context; 429 } 430 431 static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn, 432 struct qed_rdma_start_in_params *params) 433 { 434 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 435 struct qed_dev *cdev = p_hwfn->cdev; 436 u32 pci_status_control; 437 u32 num_qps; 438 439 /* Vendor specific information */ 440 dev->vendor_id = cdev->vendor_id; 441 dev->vendor_part_id = cdev->device_id; 442 dev->hw_ver = 0; 443 dev->fw_ver = (FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) | 444 (FW_REVISION_VERSION << 8) | (FW_ENGINEERING_VERSION); 445 446 qed_rdma_get_guid(p_hwfn, (u8 *)&dev->sys_image_guid); 447 dev->node_guid = dev->sys_image_guid; 448 449 dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE, 450 RDMA_MAX_SGE_PER_RQ_WQE); 451 452 if (cdev->rdma_max_sge) 453 dev->max_sge = min_t(u32, cdev->rdma_max_sge, dev->max_sge); 454 455 dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE; 456 457 dev->max_inline = (cdev->rdma_max_inline) ? 458 min_t(u32, cdev->rdma_max_inline, dev->max_inline) : 459 dev->max_inline; 460 461 dev->max_wqe = QED_RDMA_MAX_WQE; 462 dev->max_cnq = (u8)FEAT_NUM(p_hwfn, QED_RDMA_CNQ); 463 464 /* The number of QPs may be higher than QED_ROCE_MAX_QPS, because 465 * it is up-aligned to 16 and then to ILT page size within qed cxt. 466 * This is OK in terms of ILT but we don't want to configure the FW 467 * above its abilities 468 */ 469 num_qps = ROCE_MAX_QPS; 470 num_qps = min_t(u64, num_qps, p_hwfn->p_rdma_info->num_qps); 471 dev->max_qp = num_qps; 472 473 /* CQs uses the same icids that QPs use hence they are limited by the 474 * number of icids. There are two icids per QP. 475 */ 476 dev->max_cq = num_qps * 2; 477 478 /* The number of mrs is smaller by 1 since the first is reserved */ 479 dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1; 480 dev->max_mr_size = QED_RDMA_MAX_MR_SIZE; 481 482 /* The maximum CQE capacity per CQ supported. 483 * max number of cqes will be in two layer pbl, 484 * 8 is the pointer size in bytes 485 * 32 is the size of cq element in bytes 486 */ 487 if (params->cq_mode == QED_RDMA_CQ_MODE_32_BITS) 488 dev->max_cqe = QED_RDMA_MAX_CQE_32_BIT; 489 else 490 dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT; 491 492 dev->max_mw = 0; 493 dev->max_fmr = QED_RDMA_MAX_FMR; 494 dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8); 495 dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE; 496 dev->max_pkey = QED_RDMA_MAX_P_KEY; 497 498 dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 499 (RDMA_RESP_RD_ATOMIC_ELM_SIZE * 2); 500 dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 501 RDMA_REQ_RD_ATOMIC_ELM_SIZE; 502 dev->max_dev_resp_rd_atomic_resc = dev->max_qp_resp_rd_atomic_resc * 503 p_hwfn->p_rdma_info->num_qps; 504 dev->page_size_caps = QED_RDMA_PAGE_SIZE_CAPS; 505 dev->dev_ack_delay = QED_RDMA_ACK_DELAY; 506 dev->max_pd = RDMA_MAX_PDS; 507 dev->max_ah = p_hwfn->p_rdma_info->num_qps; 508 dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, QED_RDMA_STATS_QUEUE); 509 510 /* Set capablities */ 511 dev->dev_caps = 0; 512 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RNR_NAK, 1); 513 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1); 514 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1); 515 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RESIZE_CQ, 1); 516 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1); 517 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1); 518 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ZBVA, 1); 519 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1); 520 521 /* Check atomic operations support in PCI configuration space. */ 522 pci_read_config_dword(cdev->pdev, 523 cdev->pdev->pcie_cap + PCI_EXP_DEVCTL2, 524 &pci_status_control); 525 526 if (pci_status_control & PCI_EXP_DEVCTL2_LTR_EN) 527 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ATOMIC_OP, 1); 528 } 529 530 static void qed_rdma_init_port(struct qed_hwfn *p_hwfn) 531 { 532 struct qed_rdma_port *port = p_hwfn->p_rdma_info->port; 533 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 534 535 port->port_state = p_hwfn->mcp_info->link_output.link_up ? 536 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 537 538 port->max_msg_size = min_t(u64, 539 (dev->max_mr_mw_fmr_size * 540 p_hwfn->cdev->rdma_max_sge), 541 BIT(31)); 542 543 port->pkey_bad_counter = 0; 544 } 545 546 static int qed_rdma_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 547 { 548 u32 ll2_ethertype_en; 549 550 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW\n"); 551 p_hwfn->b_rdma_enabled_in_prs = false; 552 553 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 554 555 p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE; 556 557 /* We delay writing to this reg until first cid is allocated. See 558 * qed_cxt_dynamic_ilt_alloc function for more details 559 */ 560 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 561 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 562 (ll2_ethertype_en | 0x01)); 563 564 if (qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) { 565 DP_NOTICE(p_hwfn, "The first RoCE's cid should be even\n"); 566 return -EINVAL; 567 } 568 569 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW - Done\n"); 570 return 0; 571 } 572 573 static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn, 574 struct qed_rdma_start_in_params *params, 575 struct qed_ptt *p_ptt) 576 { 577 struct rdma_init_func_ramrod_data *p_ramrod; 578 struct qed_rdma_cnq_params *p_cnq_pbl_list; 579 struct rdma_init_func_hdr *p_params_header; 580 struct rdma_cnq_params *p_cnq_params; 581 struct qed_sp_init_data init_data; 582 struct qed_spq_entry *p_ent; 583 u32 cnq_id, sb_id; 584 u16 igu_sb_id; 585 int rc; 586 587 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Starting FW\n"); 588 589 /* Save the number of cnqs for the function close ramrod */ 590 p_hwfn->p_rdma_info->num_cnqs = params->desired_cnq; 591 592 /* Get SPQ entry */ 593 memset(&init_data, 0, sizeof(init_data)); 594 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 595 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 596 597 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT, 598 p_hwfn->p_rdma_info->proto, &init_data); 599 if (rc) 600 return rc; 601 602 p_ramrod = &p_ent->ramrod.roce_init_func.rdma; 603 604 p_params_header = &p_ramrod->params_header; 605 p_params_header->cnq_start_offset = (u8)RESC_START(p_hwfn, 606 QED_RDMA_CNQ_RAM); 607 p_params_header->num_cnqs = params->desired_cnq; 608 609 if (params->cq_mode == QED_RDMA_CQ_MODE_16_BITS) 610 p_params_header->cq_ring_mode = 1; 611 else 612 p_params_header->cq_ring_mode = 0; 613 614 for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) { 615 sb_id = qed_rdma_get_sb_id(p_hwfn, cnq_id); 616 igu_sb_id = qed_get_igu_sb_id(p_hwfn, sb_id); 617 p_ramrod->cnq_params[cnq_id].sb_num = cpu_to_le16(igu_sb_id); 618 p_cnq_params = &p_ramrod->cnq_params[cnq_id]; 619 p_cnq_pbl_list = ¶ms->cnq_pbl_list[cnq_id]; 620 621 p_cnq_params->sb_index = p_hwfn->pf_params.rdma_pf_params.gl_pi; 622 p_cnq_params->num_pbl_pages = p_cnq_pbl_list->num_pbl_pages; 623 624 DMA_REGPAIR_LE(p_cnq_params->pbl_base_addr, 625 p_cnq_pbl_list->pbl_ptr); 626 627 /* we assume here that cnq_id and qz_offset are the same */ 628 p_cnq_params->queue_zone_num = 629 cpu_to_le16(p_hwfn->p_rdma_info->queue_zone_base + 630 cnq_id); 631 } 632 633 return qed_spq_post(p_hwfn, p_ent, NULL); 634 } 635 636 static int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid) 637 { 638 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 639 int rc; 640 641 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n"); 642 643 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 644 rc = qed_rdma_bmap_alloc_id(p_hwfn, 645 &p_hwfn->p_rdma_info->tid_map, itid); 646 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 647 if (rc) 648 goto out; 649 650 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid); 651 out: 652 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); 653 return rc; 654 } 655 656 static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn) 657 { 658 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 659 660 /* The first DPI is reserved for the Kernel */ 661 __set_bit(0, p_hwfn->p_rdma_info->dpi_map.bitmap); 662 663 /* Tid 0 will be used as the key for "reserved MR". 664 * The driver should allocate memory for it so it can be loaded but no 665 * ramrod should be passed on it. 666 */ 667 qed_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey); 668 if (dev->reserved_lkey != RDMA_RESERVED_LKEY) { 669 DP_NOTICE(p_hwfn, 670 "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n"); 671 return -EINVAL; 672 } 673 674 return 0; 675 } 676 677 static int qed_rdma_setup(struct qed_hwfn *p_hwfn, 678 struct qed_ptt *p_ptt, 679 struct qed_rdma_start_in_params *params) 680 { 681 int rc; 682 683 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA setup\n"); 684 685 spin_lock_init(&p_hwfn->p_rdma_info->lock); 686 687 qed_rdma_init_devinfo(p_hwfn, params); 688 qed_rdma_init_port(p_hwfn); 689 qed_rdma_init_events(p_hwfn, params); 690 691 rc = qed_rdma_reserve_lkey(p_hwfn); 692 if (rc) 693 return rc; 694 695 rc = qed_rdma_init_hw(p_hwfn, p_ptt); 696 if (rc) 697 return rc; 698 699 return qed_rdma_start_fw(p_hwfn, params, p_ptt); 700 } 701 702 static int qed_rdma_stop(void *rdma_cxt) 703 { 704 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 705 struct rdma_close_func_ramrod_data *p_ramrod; 706 struct qed_sp_init_data init_data; 707 struct qed_spq_entry *p_ent; 708 struct qed_ptt *p_ptt; 709 u32 ll2_ethertype_en; 710 int rc = -EBUSY; 711 712 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop\n"); 713 714 p_ptt = qed_ptt_acquire(p_hwfn); 715 if (!p_ptt) { 716 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Failed to acquire PTT\n"); 717 return rc; 718 } 719 720 /* Disable RoCE search */ 721 qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0); 722 p_hwfn->b_rdma_enabled_in_prs = false; 723 724 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 725 726 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 727 728 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 729 (ll2_ethertype_en & 0xFFFE)); 730 731 qed_ptt_release(p_hwfn, p_ptt); 732 733 /* Get SPQ entry */ 734 memset(&init_data, 0, sizeof(init_data)); 735 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 736 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 737 738 /* Stop RoCE */ 739 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE, 740 p_hwfn->p_rdma_info->proto, &init_data); 741 if (rc) 742 goto out; 743 744 p_ramrod = &p_ent->ramrod.rdma_close_func; 745 746 p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs; 747 p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM); 748 749 rc = qed_spq_post(p_hwfn, p_ent, NULL); 750 751 out: 752 qed_rdma_free(p_hwfn); 753 754 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop done, rc = %d\n", rc); 755 return rc; 756 } 757 758 static int qed_rdma_add_user(void *rdma_cxt, 759 struct qed_rdma_add_user_out_params *out_params) 760 { 761 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 762 u32 dpi_start_offset; 763 u32 returned_id = 0; 764 int rc; 765 766 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding User\n"); 767 768 /* Allocate DPI */ 769 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 770 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 771 &returned_id); 772 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 773 774 out_params->dpi = (u16)returned_id; 775 776 /* Calculate the corresponding DPI address */ 777 dpi_start_offset = p_hwfn->dpi_start_offset; 778 779 out_params->dpi_addr = (u64)((u8 __iomem *)p_hwfn->doorbells + 780 dpi_start_offset + 781 ((out_params->dpi) * p_hwfn->dpi_size)); 782 783 out_params->dpi_phys_addr = p_hwfn->cdev->db_phys_addr + 784 dpi_start_offset + 785 ((out_params->dpi) * p_hwfn->dpi_size); 786 787 out_params->dpi_size = p_hwfn->dpi_size; 788 out_params->wid_count = p_hwfn->wid_count; 789 790 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding user - done, rc = %d\n", rc); 791 return rc; 792 } 793 794 static struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) 795 { 796 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 797 struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port; 798 799 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA Query port\n"); 800 801 /* Link may have changed */ 802 p_port->port_state = p_hwfn->mcp_info->link_output.link_up ? 803 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 804 805 p_port->link_speed = p_hwfn->mcp_info->link_output.speed; 806 807 p_port->max_msg_size = RDMA_MAX_DATA_SIZE_IN_WQE; 808 809 return p_port; 810 } 811 812 static struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) 813 { 814 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 815 816 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query device\n"); 817 818 /* Return struct with device parameters */ 819 return p_hwfn->p_rdma_info->dev; 820 } 821 822 static void qed_rdma_free_tid(void *rdma_cxt, u32 itid) 823 { 824 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 825 826 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 827 828 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 829 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid); 830 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 831 } 832 833 static void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) 834 { 835 struct qed_hwfn *p_hwfn; 836 u16 qz_num; 837 u32 addr; 838 839 p_hwfn = (struct qed_hwfn *)rdma_cxt; 840 841 if (qz_offset > p_hwfn->p_rdma_info->max_queue_zones) { 842 DP_NOTICE(p_hwfn, 843 "queue zone offset %d is too large (max is %d)\n", 844 qz_offset, p_hwfn->p_rdma_info->max_queue_zones); 845 return; 846 } 847 848 qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset; 849 addr = GTT_BAR0_MAP_REG_USDM_RAM + 850 USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num); 851 852 REG_WR16(p_hwfn, addr, prod); 853 854 /* keep prod updates ordered */ 855 wmb(); 856 } 857 858 static int qed_fill_rdma_dev_info(struct qed_dev *cdev, 859 struct qed_dev_rdma_info *info) 860 { 861 struct qed_hwfn *p_hwfn = QED_LEADING_HWFN(cdev); 862 863 memset(info, 0, sizeof(*info)); 864 865 info->rdma_type = QED_RDMA_TYPE_ROCE; 866 info->user_dpm_enabled = (p_hwfn->db_bar_no_edpm == 0); 867 868 qed_fill_dev_info(cdev, &info->common); 869 870 return 0; 871 } 872 873 static int qed_rdma_get_sb_start(struct qed_dev *cdev) 874 { 875 int feat_num; 876 877 if (cdev->num_hwfns > 1) 878 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE); 879 else 880 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE) * 881 cdev->num_hwfns; 882 883 return feat_num; 884 } 885 886 static int qed_rdma_get_min_cnq_msix(struct qed_dev *cdev) 887 { 888 int n_cnq = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_RDMA_CNQ); 889 int n_msix = cdev->int_params.rdma_msix_cnt; 890 891 return min_t(int, n_cnq, n_msix); 892 } 893 894 static int qed_rdma_set_int(struct qed_dev *cdev, u16 cnt) 895 { 896 int limit = 0; 897 898 /* Mark the fastpath as free/used */ 899 cdev->int_params.fp_initialized = cnt ? true : false; 900 901 if (cdev->int_params.out.int_mode != QED_INT_MODE_MSIX) { 902 DP_ERR(cdev, 903 "qed roce supports only MSI-X interrupts (detected %d).\n", 904 cdev->int_params.out.int_mode); 905 return -EINVAL; 906 } else if (cdev->int_params.fp_msix_cnt) { 907 limit = cdev->int_params.rdma_msix_cnt; 908 } 909 910 if (!limit) 911 return -ENOMEM; 912 913 return min_t(int, cnt, limit); 914 } 915 916 static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info) 917 { 918 memset(info, 0, sizeof(*info)); 919 920 if (!cdev->int_params.fp_initialized) { 921 DP_INFO(cdev, 922 "Protocol driver requested interrupt information, but its support is not yet configured\n"); 923 return -EINVAL; 924 } 925 926 if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) { 927 int msix_base = cdev->int_params.rdma_msix_base; 928 929 info->msix_cnt = cdev->int_params.rdma_msix_cnt; 930 info->msix = &cdev->int_params.msix_table[msix_base]; 931 932 DP_VERBOSE(cdev, QED_MSG_RDMA, "msix_cnt = %d msix_base=%d\n", 933 info->msix_cnt, msix_base); 934 } 935 936 return 0; 937 } 938 939 static int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) 940 { 941 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 942 u32 returned_id; 943 int rc; 944 945 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD\n"); 946 947 /* Allocates an unused protection domain */ 948 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 949 rc = qed_rdma_bmap_alloc_id(p_hwfn, 950 &p_hwfn->p_rdma_info->pd_map, &returned_id); 951 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 952 953 *pd = (u16)returned_id; 954 955 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc); 956 return rc; 957 } 958 959 static void qed_rdma_free_pd(void *rdma_cxt, u16 pd) 960 { 961 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 962 963 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "pd = %08x\n", pd); 964 965 /* Returns a previously allocated protection domain for reuse */ 966 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 967 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd); 968 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 969 } 970 971 static enum qed_rdma_toggle_bit 972 qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid) 973 { 974 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 975 enum qed_rdma_toggle_bit toggle_bit; 976 u32 bmap_id; 977 978 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", icid); 979 980 /* the function toggle the bit that is related to a given icid 981 * and returns the new toggle bit's value 982 */ 983 bmap_id = icid - qed_cxt_get_proto_cid_start(p_hwfn, p_info->proto); 984 985 spin_lock_bh(&p_info->lock); 986 toggle_bit = !test_and_change_bit(bmap_id, 987 p_info->toggle_bits.bitmap); 988 spin_unlock_bh(&p_info->lock); 989 990 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QED_RDMA_TOGGLE_BIT_= %d\n", 991 toggle_bit); 992 993 return toggle_bit; 994 } 995 996 static int qed_rdma_create_cq(void *rdma_cxt, 997 struct qed_rdma_create_cq_in_params *params, 998 u16 *icid) 999 { 1000 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1001 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 1002 struct rdma_create_cq_ramrod_data *p_ramrod; 1003 enum qed_rdma_toggle_bit toggle_bit; 1004 struct qed_sp_init_data init_data; 1005 struct qed_spq_entry *p_ent; 1006 u32 returned_id, start_cid; 1007 int rc; 1008 1009 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "cq_handle = %08x%08x\n", 1010 params->cq_handle_hi, params->cq_handle_lo); 1011 1012 /* Allocate icid */ 1013 spin_lock_bh(&p_info->lock); 1014 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_info->cq_map, &returned_id); 1015 spin_unlock_bh(&p_info->lock); 1016 1017 if (rc) { 1018 DP_NOTICE(p_hwfn, "Can't create CQ, rc = %d\n", rc); 1019 return rc; 1020 } 1021 1022 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, 1023 p_info->proto); 1024 *icid = returned_id + start_cid; 1025 1026 /* Check if icid requires a page allocation */ 1027 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *icid); 1028 if (rc) 1029 goto err; 1030 1031 /* Get SPQ entry */ 1032 memset(&init_data, 0, sizeof(init_data)); 1033 init_data.cid = *icid; 1034 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1035 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1036 1037 /* Send create CQ ramrod */ 1038 rc = qed_sp_init_request(p_hwfn, &p_ent, 1039 RDMA_RAMROD_CREATE_CQ, 1040 p_info->proto, &init_data); 1041 if (rc) 1042 goto err; 1043 1044 p_ramrod = &p_ent->ramrod.rdma_create_cq; 1045 1046 p_ramrod->cq_handle.hi = cpu_to_le32(params->cq_handle_hi); 1047 p_ramrod->cq_handle.lo = cpu_to_le32(params->cq_handle_lo); 1048 p_ramrod->dpi = cpu_to_le16(params->dpi); 1049 p_ramrod->is_two_level_pbl = params->pbl_two_level; 1050 p_ramrod->max_cqes = cpu_to_le32(params->cq_size); 1051 DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr); 1052 p_ramrod->pbl_num_pages = cpu_to_le16(params->pbl_num_pages); 1053 p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM) + 1054 params->cnq_id; 1055 p_ramrod->int_timeout = params->int_timeout; 1056 1057 /* toggle the bit for every resize or create cq for a given icid */ 1058 toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 1059 1060 p_ramrod->toggle_bit = toggle_bit; 1061 1062 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1063 if (rc) { 1064 /* restore toggle bit */ 1065 qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 1066 goto err; 1067 } 1068 1069 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Created CQ, rc = %d\n", rc); 1070 return rc; 1071 1072 err: 1073 /* release allocated icid */ 1074 spin_lock_bh(&p_info->lock); 1075 qed_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id); 1076 spin_unlock_bh(&p_info->lock); 1077 DP_NOTICE(p_hwfn, "Create CQ failed, rc = %d\n", rc); 1078 1079 return rc; 1080 } 1081 1082 static int 1083 qed_rdma_destroy_cq(void *rdma_cxt, 1084 struct qed_rdma_destroy_cq_in_params *in_params, 1085 struct qed_rdma_destroy_cq_out_params *out_params) 1086 { 1087 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1088 struct rdma_destroy_cq_output_params *p_ramrod_res; 1089 struct rdma_destroy_cq_ramrod_data *p_ramrod; 1090 struct qed_sp_init_data init_data; 1091 struct qed_spq_entry *p_ent; 1092 dma_addr_t ramrod_res_phys; 1093 int rc = -ENOMEM; 1094 1095 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid); 1096 1097 p_ramrod_res = 1098 (struct rdma_destroy_cq_output_params *) 1099 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1100 sizeof(struct rdma_destroy_cq_output_params), 1101 &ramrod_res_phys, GFP_KERNEL); 1102 if (!p_ramrod_res) { 1103 DP_NOTICE(p_hwfn, 1104 "qed destroy cq failed: cannot allocate memory (ramrod)\n"); 1105 return rc; 1106 } 1107 1108 /* Get SPQ entry */ 1109 memset(&init_data, 0, sizeof(init_data)); 1110 init_data.cid = in_params->icid; 1111 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1112 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1113 1114 /* Send destroy CQ ramrod */ 1115 rc = qed_sp_init_request(p_hwfn, &p_ent, 1116 RDMA_RAMROD_DESTROY_CQ, 1117 p_hwfn->p_rdma_info->proto, &init_data); 1118 if (rc) 1119 goto err; 1120 1121 p_ramrod = &p_ent->ramrod.rdma_destroy_cq; 1122 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1123 1124 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1125 if (rc) 1126 goto err; 1127 1128 out_params->num_cq_notif = le16_to_cpu(p_ramrod_res->cnq_num); 1129 1130 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1131 sizeof(struct rdma_destroy_cq_output_params), 1132 p_ramrod_res, ramrod_res_phys); 1133 1134 /* Free icid */ 1135 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1136 1137 qed_bmap_release_id(p_hwfn, 1138 &p_hwfn->p_rdma_info->cq_map, 1139 (in_params->icid - 1140 qed_cxt_get_proto_cid_start(p_hwfn, 1141 p_hwfn-> 1142 p_rdma_info->proto))); 1143 1144 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1145 1146 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc); 1147 return rc; 1148 1149 err: dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1150 sizeof(struct rdma_destroy_cq_output_params), 1151 p_ramrod_res, ramrod_res_phys); 1152 1153 return rc; 1154 } 1155 1156 static void qed_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_qed_mac) 1157 { 1158 p_fw_mac[0] = cpu_to_le16((p_qed_mac[0] << 8) + p_qed_mac[1]); 1159 p_fw_mac[1] = cpu_to_le16((p_qed_mac[2] << 8) + p_qed_mac[3]); 1160 p_fw_mac[2] = cpu_to_le16((p_qed_mac[4] << 8) + p_qed_mac[5]); 1161 } 1162 1163 static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid, 1164 __le32 *dst_gid) 1165 { 1166 u32 i; 1167 1168 if (qp->roce_mode == ROCE_V2_IPV4) { 1169 /* The IPv4 addresses shall be aligned to the highest word. 1170 * The lower words must be zero. 1171 */ 1172 memset(src_gid, 0, sizeof(union qed_gid)); 1173 memset(dst_gid, 0, sizeof(union qed_gid)); 1174 src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr); 1175 dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr); 1176 } else { 1177 /* GIDs and IPv6 addresses coincide in location and size */ 1178 for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) { 1179 src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]); 1180 dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]); 1181 } 1182 } 1183 } 1184 1185 static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode) 1186 { 1187 enum roce_flavor flavor; 1188 1189 switch (roce_mode) { 1190 case ROCE_V1: 1191 flavor = PLAIN_ROCE; 1192 break; 1193 case ROCE_V2_IPV4: 1194 flavor = RROCE_IPV4; 1195 break; 1196 case ROCE_V2_IPV6: 1197 flavor = ROCE_V2_IPV6; 1198 break; 1199 default: 1200 flavor = MAX_ROCE_MODE; 1201 break; 1202 } 1203 return flavor; 1204 } 1205 1206 void qed_roce_free_cid_pair(struct qed_hwfn *p_hwfn, u16 cid) 1207 { 1208 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1209 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid); 1210 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid + 1); 1211 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1212 } 1213 1214 static int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid) 1215 { 1216 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 1217 u32 responder_icid; 1218 u32 requester_icid; 1219 int rc; 1220 1221 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1222 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1223 &responder_icid); 1224 if (rc) { 1225 spin_unlock_bh(&p_rdma_info->lock); 1226 return rc; 1227 } 1228 1229 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1230 &requester_icid); 1231 1232 spin_unlock_bh(&p_rdma_info->lock); 1233 if (rc) 1234 goto err; 1235 1236 /* the two icid's should be adjacent */ 1237 if ((requester_icid - responder_icid) != 1) { 1238 DP_NOTICE(p_hwfn, "Failed to allocate two adjacent qp's'\n"); 1239 rc = -EINVAL; 1240 goto err; 1241 } 1242 1243 responder_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1244 p_rdma_info->proto); 1245 requester_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1246 p_rdma_info->proto); 1247 1248 /* If these icids require a new ILT line allocate DMA-able context for 1249 * an ILT page 1250 */ 1251 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, responder_icid); 1252 if (rc) 1253 goto err; 1254 1255 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, requester_icid); 1256 if (rc) 1257 goto err; 1258 1259 *cid = (u16)responder_icid; 1260 return rc; 1261 1262 err: 1263 spin_lock_bh(&p_rdma_info->lock); 1264 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, responder_icid); 1265 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, requester_icid); 1266 1267 spin_unlock_bh(&p_rdma_info->lock); 1268 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1269 "Allocate CID - failed, rc = %d\n", rc); 1270 return rc; 1271 } 1272 1273 static void qed_roce_set_real_cid(struct qed_hwfn *p_hwfn, u32 cid) 1274 { 1275 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1276 qed_bmap_set_id(p_hwfn, &p_hwfn->p_rdma_info->real_cid_map, cid); 1277 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1278 } 1279 1280 static int qed_roce_sp_create_responder(struct qed_hwfn *p_hwfn, 1281 struct qed_rdma_qp *qp) 1282 { 1283 struct roce_create_qp_resp_ramrod_data *p_ramrod; 1284 struct qed_sp_init_data init_data; 1285 enum roce_flavor roce_flavor; 1286 struct qed_spq_entry *p_ent; 1287 u16 regular_latency_queue; 1288 enum protocol_type proto; 1289 int rc; 1290 1291 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1292 1293 /* Allocate DMA-able memory for IRQ */ 1294 qp->irq_num_pages = 1; 1295 qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1296 RDMA_RING_PAGE_SIZE, 1297 &qp->irq_phys_addr, GFP_KERNEL); 1298 if (!qp->irq) { 1299 rc = -ENOMEM; 1300 DP_NOTICE(p_hwfn, 1301 "qed create responder failed: cannot allocate memory (irq). rc = %d\n", 1302 rc); 1303 return rc; 1304 } 1305 1306 /* Get SPQ entry */ 1307 memset(&init_data, 0, sizeof(init_data)); 1308 init_data.cid = qp->icid; 1309 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1310 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1311 1312 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP, 1313 PROTOCOLID_ROCE, &init_data); 1314 if (rc) 1315 goto err; 1316 1317 p_ramrod = &p_ent->ramrod.roce_create_qp_resp; 1318 1319 p_ramrod->flags = 0; 1320 1321 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1322 SET_FIELD(p_ramrod->flags, 1323 ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1324 1325 SET_FIELD(p_ramrod->flags, 1326 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1327 qp->incoming_rdma_read_en); 1328 1329 SET_FIELD(p_ramrod->flags, 1330 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1331 qp->incoming_rdma_write_en); 1332 1333 SET_FIELD(p_ramrod->flags, 1334 ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1335 qp->incoming_atomic_en); 1336 1337 SET_FIELD(p_ramrod->flags, 1338 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1339 qp->e2e_flow_control_en); 1340 1341 SET_FIELD(p_ramrod->flags, 1342 ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq); 1343 1344 SET_FIELD(p_ramrod->flags, 1345 ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN, 1346 qp->fmr_and_reserved_lkey); 1347 1348 SET_FIELD(p_ramrod->flags, 1349 ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1350 qp->min_rnr_nak_timer); 1351 1352 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1353 p_ramrod->traffic_class = qp->traffic_class_tos; 1354 p_ramrod->hop_limit = qp->hop_limit_ttl; 1355 p_ramrod->irq_num_pages = qp->irq_num_pages; 1356 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1357 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1358 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1359 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1360 p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn); 1361 p_ramrod->pd = cpu_to_le16(qp->pd); 1362 p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages); 1363 DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr); 1364 DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr); 1365 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1366 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1367 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1368 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1369 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1370 p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | 1371 qp->rq_cq_id); 1372 1373 regular_latency_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); 1374 1375 p_ramrod->regular_latency_phy_queue = 1376 cpu_to_le16(regular_latency_queue); 1377 p_ramrod->low_latency_phy_queue = 1378 cpu_to_le16(regular_latency_queue); 1379 1380 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1381 1382 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1383 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1384 1385 p_ramrod->udp_src_port = qp->udp_src_port; 1386 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1387 p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id); 1388 p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid); 1389 1390 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1391 qp->stats_queue; 1392 1393 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1394 1395 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1396 "rc = %d regular physical queue = 0x%x\n", rc, 1397 regular_latency_queue); 1398 1399 if (rc) 1400 goto err; 1401 1402 qp->resp_offloaded = true; 1403 qp->cq_prod = 0; 1404 1405 proto = p_hwfn->p_rdma_info->proto; 1406 qed_roce_set_real_cid(p_hwfn, qp->icid - 1407 qed_cxt_get_proto_cid_start(p_hwfn, proto)); 1408 1409 return rc; 1410 1411 err: 1412 DP_NOTICE(p_hwfn, "create responder - failed, rc = %d\n", rc); 1413 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1414 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1415 qp->irq, qp->irq_phys_addr); 1416 1417 return rc; 1418 } 1419 1420 static int qed_roce_sp_create_requester(struct qed_hwfn *p_hwfn, 1421 struct qed_rdma_qp *qp) 1422 { 1423 struct roce_create_qp_req_ramrod_data *p_ramrod; 1424 struct qed_sp_init_data init_data; 1425 enum roce_flavor roce_flavor; 1426 struct qed_spq_entry *p_ent; 1427 u16 regular_latency_queue; 1428 enum protocol_type proto; 1429 int rc; 1430 1431 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1432 1433 /* Allocate DMA-able memory for ORQ */ 1434 qp->orq_num_pages = 1; 1435 qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1436 RDMA_RING_PAGE_SIZE, 1437 &qp->orq_phys_addr, GFP_KERNEL); 1438 if (!qp->orq) { 1439 rc = -ENOMEM; 1440 DP_NOTICE(p_hwfn, 1441 "qed create requester failed: cannot allocate memory (orq). rc = %d\n", 1442 rc); 1443 return rc; 1444 } 1445 1446 /* Get SPQ entry */ 1447 memset(&init_data, 0, sizeof(init_data)); 1448 init_data.cid = qp->icid + 1; 1449 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1450 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1451 1452 rc = qed_sp_init_request(p_hwfn, &p_ent, 1453 ROCE_RAMROD_CREATE_QP, 1454 PROTOCOLID_ROCE, &init_data); 1455 if (rc) 1456 goto err; 1457 1458 p_ramrod = &p_ent->ramrod.roce_create_qp_req; 1459 1460 p_ramrod->flags = 0; 1461 1462 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1463 SET_FIELD(p_ramrod->flags, 1464 ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1465 1466 SET_FIELD(p_ramrod->flags, 1467 ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN, 1468 qp->fmr_and_reserved_lkey); 1469 1470 SET_FIELD(p_ramrod->flags, 1471 ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); 1472 1473 SET_FIELD(p_ramrod->flags, 1474 ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1475 1476 SET_FIELD(p_ramrod->flags, 1477 ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1478 qp->rnr_retry_cnt); 1479 1480 p_ramrod->max_ord = qp->max_rd_atomic_req; 1481 p_ramrod->traffic_class = qp->traffic_class_tos; 1482 p_ramrod->hop_limit = qp->hop_limit_ttl; 1483 p_ramrod->orq_num_pages = qp->orq_num_pages; 1484 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1485 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1486 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1487 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1488 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1489 p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn); 1490 p_ramrod->pd = cpu_to_le16(qp->pd); 1491 p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages); 1492 DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr); 1493 DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr); 1494 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1495 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1496 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1497 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1498 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1499 p_ramrod->cq_cid = 1500 cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id); 1501 1502 regular_latency_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); 1503 1504 p_ramrod->regular_latency_phy_queue = 1505 cpu_to_le16(regular_latency_queue); 1506 p_ramrod->low_latency_phy_queue = 1507 cpu_to_le16(regular_latency_queue); 1508 1509 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1510 1511 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1512 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1513 1514 p_ramrod->udp_src_port = qp->udp_src_port; 1515 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1516 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1517 qp->stats_queue; 1518 1519 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1520 1521 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 1522 1523 if (rc) 1524 goto err; 1525 1526 qp->req_offloaded = true; 1527 proto = p_hwfn->p_rdma_info->proto; 1528 qed_roce_set_real_cid(p_hwfn, 1529 qp->icid + 1 - 1530 qed_cxt_get_proto_cid_start(p_hwfn, proto)); 1531 1532 return rc; 1533 1534 err: 1535 DP_NOTICE(p_hwfn, "Create requested - failed, rc = %d\n", rc); 1536 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1537 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1538 qp->orq, qp->orq_phys_addr); 1539 return rc; 1540 } 1541 1542 static int qed_roce_sp_modify_responder(struct qed_hwfn *p_hwfn, 1543 struct qed_rdma_qp *qp, 1544 bool move_to_err, u32 modify_flags) 1545 { 1546 struct roce_modify_qp_resp_ramrod_data *p_ramrod; 1547 struct qed_sp_init_data init_data; 1548 struct qed_spq_entry *p_ent; 1549 int rc; 1550 1551 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1552 1553 if (move_to_err && !qp->resp_offloaded) 1554 return 0; 1555 1556 /* Get SPQ entry */ 1557 memset(&init_data, 0, sizeof(init_data)); 1558 init_data.cid = qp->icid; 1559 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1560 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1561 1562 rc = qed_sp_init_request(p_hwfn, &p_ent, 1563 ROCE_EVENT_MODIFY_QP, 1564 PROTOCOLID_ROCE, &init_data); 1565 if (rc) { 1566 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1567 return rc; 1568 } 1569 1570 p_ramrod = &p_ent->ramrod.roce_modify_qp_resp; 1571 1572 p_ramrod->flags = 0; 1573 1574 SET_FIELD(p_ramrod->flags, 1575 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1576 1577 SET_FIELD(p_ramrod->flags, 1578 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1579 qp->incoming_rdma_read_en); 1580 1581 SET_FIELD(p_ramrod->flags, 1582 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1583 qp->incoming_rdma_write_en); 1584 1585 SET_FIELD(p_ramrod->flags, 1586 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1587 qp->incoming_atomic_en); 1588 1589 SET_FIELD(p_ramrod->flags, 1590 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1591 qp->e2e_flow_control_en); 1592 1593 SET_FIELD(p_ramrod->flags, 1594 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG, 1595 GET_FIELD(modify_flags, 1596 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)); 1597 1598 SET_FIELD(p_ramrod->flags, 1599 ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG, 1600 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1601 1602 SET_FIELD(p_ramrod->flags, 1603 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1604 GET_FIELD(modify_flags, 1605 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1606 1607 SET_FIELD(p_ramrod->flags, 1608 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG, 1609 GET_FIELD(modify_flags, 1610 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)); 1611 1612 SET_FIELD(p_ramrod->flags, 1613 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG, 1614 GET_FIELD(modify_flags, 1615 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)); 1616 1617 p_ramrod->fields = 0; 1618 SET_FIELD(p_ramrod->fields, 1619 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1620 qp->min_rnr_nak_timer); 1621 1622 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1623 p_ramrod->traffic_class = qp->traffic_class_tos; 1624 p_ramrod->hop_limit = qp->hop_limit_ttl; 1625 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1626 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1627 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1628 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1629 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1630 1631 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify responder, rc = %d\n", rc); 1632 return rc; 1633 } 1634 1635 static int qed_roce_sp_modify_requester(struct qed_hwfn *p_hwfn, 1636 struct qed_rdma_qp *qp, 1637 bool move_to_sqd, 1638 bool move_to_err, u32 modify_flags) 1639 { 1640 struct roce_modify_qp_req_ramrod_data *p_ramrod; 1641 struct qed_sp_init_data init_data; 1642 struct qed_spq_entry *p_ent; 1643 int rc; 1644 1645 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1646 1647 if (move_to_err && !(qp->req_offloaded)) 1648 return 0; 1649 1650 /* Get SPQ entry */ 1651 memset(&init_data, 0, sizeof(init_data)); 1652 init_data.cid = qp->icid + 1; 1653 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1654 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1655 1656 rc = qed_sp_init_request(p_hwfn, &p_ent, 1657 ROCE_EVENT_MODIFY_QP, 1658 PROTOCOLID_ROCE, &init_data); 1659 if (rc) { 1660 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1661 return rc; 1662 } 1663 1664 p_ramrod = &p_ent->ramrod.roce_modify_qp_req; 1665 1666 p_ramrod->flags = 0; 1667 1668 SET_FIELD(p_ramrod->flags, 1669 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1670 1671 SET_FIELD(p_ramrod->flags, 1672 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, move_to_sqd); 1673 1674 SET_FIELD(p_ramrod->flags, 1675 ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY, 1676 qp->sqd_async); 1677 1678 SET_FIELD(p_ramrod->flags, 1679 ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG, 1680 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1681 1682 SET_FIELD(p_ramrod->flags, 1683 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1684 GET_FIELD(modify_flags, 1685 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1686 1687 SET_FIELD(p_ramrod->flags, 1688 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG, 1689 GET_FIELD(modify_flags, 1690 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)); 1691 1692 SET_FIELD(p_ramrod->flags, 1693 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG, 1694 GET_FIELD(modify_flags, 1695 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)); 1696 1697 SET_FIELD(p_ramrod->flags, 1698 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG, 1699 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)); 1700 1701 SET_FIELD(p_ramrod->flags, 1702 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG, 1703 GET_FIELD(modify_flags, 1704 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)); 1705 1706 p_ramrod->fields = 0; 1707 SET_FIELD(p_ramrod->fields, 1708 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1709 1710 SET_FIELD(p_ramrod->fields, 1711 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1712 qp->rnr_retry_cnt); 1713 1714 p_ramrod->max_ord = qp->max_rd_atomic_req; 1715 p_ramrod->traffic_class = qp->traffic_class_tos; 1716 p_ramrod->hop_limit = qp->hop_limit_ttl; 1717 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1718 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1719 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1720 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1721 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1722 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1723 1724 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify requester, rc = %d\n", rc); 1725 return rc; 1726 } 1727 1728 static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn *p_hwfn, 1729 struct qed_rdma_qp *qp, 1730 u32 *num_invalidated_mw, 1731 u32 *cq_prod) 1732 { 1733 struct roce_destroy_qp_resp_output_params *p_ramrod_res; 1734 struct roce_destroy_qp_resp_ramrod_data *p_ramrod; 1735 struct qed_sp_init_data init_data; 1736 struct qed_spq_entry *p_ent; 1737 dma_addr_t ramrod_res_phys; 1738 int rc; 1739 1740 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1741 1742 *num_invalidated_mw = 0; 1743 *cq_prod = qp->cq_prod; 1744 1745 if (!qp->resp_offloaded) { 1746 /* If a responder was never offload, we need to free the cids 1747 * allocated in create_qp as a FW async event will never arrive 1748 */ 1749 u32 cid; 1750 1751 cid = qp->icid - 1752 qed_cxt_get_proto_cid_start(p_hwfn, 1753 p_hwfn->p_rdma_info->proto); 1754 qed_roce_free_cid_pair(p_hwfn, (u16)cid); 1755 1756 return 0; 1757 } 1758 1759 /* Get SPQ entry */ 1760 memset(&init_data, 0, sizeof(init_data)); 1761 init_data.cid = qp->icid; 1762 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1763 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1764 1765 rc = qed_sp_init_request(p_hwfn, &p_ent, 1766 ROCE_RAMROD_DESTROY_QP, 1767 PROTOCOLID_ROCE, &init_data); 1768 if (rc) 1769 return rc; 1770 1771 p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp; 1772 1773 p_ramrod_res = (struct roce_destroy_qp_resp_output_params *) 1774 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1775 &ramrod_res_phys, GFP_KERNEL); 1776 1777 if (!p_ramrod_res) { 1778 rc = -ENOMEM; 1779 DP_NOTICE(p_hwfn, 1780 "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n", 1781 rc); 1782 return rc; 1783 } 1784 1785 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1786 1787 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1788 if (rc) 1789 goto err; 1790 1791 *num_invalidated_mw = le32_to_cpu(p_ramrod_res->num_invalidated_mw); 1792 *cq_prod = le32_to_cpu(p_ramrod_res->cq_prod); 1793 qp->cq_prod = *cq_prod; 1794 1795 /* Free IRQ - only if ramrod succeeded, in case FW is still using it */ 1796 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1797 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1798 qp->irq, qp->irq_phys_addr); 1799 1800 qp->resp_offloaded = false; 1801 1802 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy responder, rc = %d\n", rc); 1803 1804 err: 1805 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1806 sizeof(struct roce_destroy_qp_resp_output_params), 1807 p_ramrod_res, ramrod_res_phys); 1808 1809 return rc; 1810 } 1811 1812 static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn *p_hwfn, 1813 struct qed_rdma_qp *qp, 1814 u32 *num_bound_mw) 1815 { 1816 struct roce_destroy_qp_req_output_params *p_ramrod_res; 1817 struct roce_destroy_qp_req_ramrod_data *p_ramrod; 1818 struct qed_sp_init_data init_data; 1819 struct qed_spq_entry *p_ent; 1820 dma_addr_t ramrod_res_phys; 1821 int rc = -ENOMEM; 1822 1823 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1824 1825 if (!qp->req_offloaded) 1826 return 0; 1827 1828 p_ramrod_res = (struct roce_destroy_qp_req_output_params *) 1829 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1830 sizeof(*p_ramrod_res), 1831 &ramrod_res_phys, GFP_KERNEL); 1832 if (!p_ramrod_res) { 1833 DP_NOTICE(p_hwfn, 1834 "qed destroy requester failed: cannot allocate memory (ramrod)\n"); 1835 return rc; 1836 } 1837 1838 /* Get SPQ entry */ 1839 memset(&init_data, 0, sizeof(init_data)); 1840 init_data.cid = qp->icid + 1; 1841 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1842 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1843 1844 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP, 1845 PROTOCOLID_ROCE, &init_data); 1846 if (rc) 1847 goto err; 1848 1849 p_ramrod = &p_ent->ramrod.roce_destroy_qp_req; 1850 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1851 1852 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1853 if (rc) 1854 goto err; 1855 1856 *num_bound_mw = le32_to_cpu(p_ramrod_res->num_bound_mw); 1857 1858 /* Free ORQ - only if ramrod succeeded, in case FW is still using it */ 1859 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1860 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1861 qp->orq, qp->orq_phys_addr); 1862 1863 qp->req_offloaded = false; 1864 1865 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy requester, rc = %d\n", rc); 1866 1867 err: 1868 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1869 p_ramrod_res, ramrod_res_phys); 1870 1871 return rc; 1872 } 1873 1874 static int qed_roce_query_qp(struct qed_hwfn *p_hwfn, 1875 struct qed_rdma_qp *qp, 1876 struct qed_rdma_query_qp_out_params *out_params) 1877 { 1878 struct roce_query_qp_resp_output_params *p_resp_ramrod_res; 1879 struct roce_query_qp_req_output_params *p_req_ramrod_res; 1880 struct roce_query_qp_resp_ramrod_data *p_resp_ramrod; 1881 struct roce_query_qp_req_ramrod_data *p_req_ramrod; 1882 struct qed_sp_init_data init_data; 1883 dma_addr_t resp_ramrod_res_phys; 1884 dma_addr_t req_ramrod_res_phys; 1885 struct qed_spq_entry *p_ent; 1886 bool rq_err_state; 1887 bool sq_err_state; 1888 bool sq_draining; 1889 int rc = -ENOMEM; 1890 1891 if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) { 1892 /* We can't send ramrod to the fw since this qp wasn't offloaded 1893 * to the fw yet 1894 */ 1895 out_params->draining = false; 1896 out_params->rq_psn = qp->rq_psn; 1897 out_params->sq_psn = qp->sq_psn; 1898 out_params->state = qp->cur_state; 1899 1900 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "No QPs as no offload\n"); 1901 return 0; 1902 } 1903 1904 if (!(qp->resp_offloaded)) { 1905 DP_NOTICE(p_hwfn, 1906 "The responder's qp should be offloded before requester's\n"); 1907 return -EINVAL; 1908 } 1909 1910 /* Send a query responder ramrod to FW to get RQ-PSN and state */ 1911 p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *) 1912 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1913 sizeof(*p_resp_ramrod_res), 1914 &resp_ramrod_res_phys, GFP_KERNEL); 1915 if (!p_resp_ramrod_res) { 1916 DP_NOTICE(p_hwfn, 1917 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1918 return rc; 1919 } 1920 1921 /* Get SPQ entry */ 1922 memset(&init_data, 0, sizeof(init_data)); 1923 init_data.cid = qp->icid; 1924 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1925 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1926 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1927 PROTOCOLID_ROCE, &init_data); 1928 if (rc) 1929 goto err_resp; 1930 1931 p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp; 1932 DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys); 1933 1934 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1935 if (rc) 1936 goto err_resp; 1937 1938 out_params->rq_psn = le32_to_cpu(p_resp_ramrod_res->psn); 1939 rq_err_state = GET_FIELD(le32_to_cpu(p_resp_ramrod_res->err_flag), 1940 ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG); 1941 1942 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 1943 p_resp_ramrod_res, resp_ramrod_res_phys); 1944 1945 if (!(qp->req_offloaded)) { 1946 /* Don't send query qp for the requester */ 1947 out_params->sq_psn = qp->sq_psn; 1948 out_params->draining = false; 1949 1950 if (rq_err_state) 1951 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1952 1953 out_params->state = qp->cur_state; 1954 1955 return 0; 1956 } 1957 1958 /* Send a query requester ramrod to FW to get SQ-PSN and state */ 1959 p_req_ramrod_res = (struct roce_query_qp_req_output_params *) 1960 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1961 sizeof(*p_req_ramrod_res), 1962 &req_ramrod_res_phys, 1963 GFP_KERNEL); 1964 if (!p_req_ramrod_res) { 1965 rc = -ENOMEM; 1966 DP_NOTICE(p_hwfn, 1967 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1968 return rc; 1969 } 1970 1971 /* Get SPQ entry */ 1972 init_data.cid = qp->icid + 1; 1973 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1974 PROTOCOLID_ROCE, &init_data); 1975 if (rc) 1976 goto err_req; 1977 1978 p_req_ramrod = &p_ent->ramrod.roce_query_qp_req; 1979 DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys); 1980 1981 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1982 if (rc) 1983 goto err_req; 1984 1985 out_params->sq_psn = le32_to_cpu(p_req_ramrod_res->psn); 1986 sq_err_state = GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1987 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG); 1988 sq_draining = 1989 GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1990 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG); 1991 1992 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 1993 p_req_ramrod_res, req_ramrod_res_phys); 1994 1995 out_params->draining = false; 1996 1997 if (rq_err_state || sq_err_state) 1998 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1999 else if (sq_draining) 2000 out_params->draining = true; 2001 out_params->state = qp->cur_state; 2002 2003 return 0; 2004 2005 err_req: 2006 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 2007 p_req_ramrod_res, req_ramrod_res_phys); 2008 return rc; 2009 err_resp: 2010 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 2011 p_resp_ramrod_res, resp_ramrod_res_phys); 2012 return rc; 2013 } 2014 2015 static int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 2016 { 2017 u32 num_invalidated_mw = 0; 2018 u32 num_bound_mw = 0; 2019 u32 cq_prod; 2020 int rc; 2021 2022 /* Destroys the specified QP */ 2023 if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) && 2024 (qp->cur_state != QED_ROCE_QP_STATE_ERR) && 2025 (qp->cur_state != QED_ROCE_QP_STATE_INIT)) { 2026 DP_NOTICE(p_hwfn, 2027 "QP must be in error, reset or init state before destroying it\n"); 2028 return -EINVAL; 2029 } 2030 2031 if (qp->cur_state != QED_ROCE_QP_STATE_RESET) { 2032 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, 2033 &num_invalidated_mw, 2034 &cq_prod); 2035 if (rc) 2036 return rc; 2037 2038 /* Send destroy requester ramrod */ 2039 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, 2040 &num_bound_mw); 2041 if (rc) 2042 return rc; 2043 2044 if (num_invalidated_mw != num_bound_mw) { 2045 DP_NOTICE(p_hwfn, 2046 "number of invalidate memory windows is different from bounded ones\n"); 2047 return -EINVAL; 2048 } 2049 } 2050 2051 return 0; 2052 } 2053 2054 static int qed_rdma_query_qp(void *rdma_cxt, 2055 struct qed_rdma_qp *qp, 2056 struct qed_rdma_query_qp_out_params *out_params) 2057 { 2058 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2059 int rc; 2060 2061 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 2062 2063 /* The following fields are filled in from qp and not FW as they can't 2064 * be modified by FW 2065 */ 2066 out_params->mtu = qp->mtu; 2067 out_params->dest_qp = qp->dest_qp; 2068 out_params->incoming_atomic_en = qp->incoming_atomic_en; 2069 out_params->e2e_flow_control_en = qp->e2e_flow_control_en; 2070 out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en; 2071 out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en; 2072 out_params->dgid = qp->dgid; 2073 out_params->flow_label = qp->flow_label; 2074 out_params->hop_limit_ttl = qp->hop_limit_ttl; 2075 out_params->traffic_class_tos = qp->traffic_class_tos; 2076 out_params->timeout = qp->ack_timeout; 2077 out_params->rnr_retry = qp->rnr_retry_cnt; 2078 out_params->retry_cnt = qp->retry_cnt; 2079 out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer; 2080 out_params->pkey_index = 0; 2081 out_params->max_rd_atomic = qp->max_rd_atomic_req; 2082 out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp; 2083 out_params->sqd_async = qp->sqd_async; 2084 2085 rc = qed_roce_query_qp(p_hwfn, qp, out_params); 2086 2087 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query QP, rc = %d\n", rc); 2088 return rc; 2089 } 2090 2091 static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) 2092 { 2093 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2094 int rc = 0; 2095 2096 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 2097 2098 rc = qed_roce_destroy_qp(p_hwfn, qp); 2099 2100 /* free qp params struct */ 2101 kfree(qp); 2102 2103 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP destroyed\n"); 2104 return rc; 2105 } 2106 2107 static struct qed_rdma_qp * 2108 qed_rdma_create_qp(void *rdma_cxt, 2109 struct qed_rdma_create_qp_in_params *in_params, 2110 struct qed_rdma_create_qp_out_params *out_params) 2111 { 2112 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2113 struct qed_rdma_qp *qp; 2114 u8 max_stats_queues; 2115 int rc; 2116 2117 if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) { 2118 DP_ERR(p_hwfn->cdev, 2119 "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n", 2120 rdma_cxt, in_params, out_params); 2121 return NULL; 2122 } 2123 2124 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2125 "qed rdma create qp called with qp_handle = %08x%08x\n", 2126 in_params->qp_handle_hi, in_params->qp_handle_lo); 2127 2128 /* Some sanity checks... */ 2129 max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues; 2130 if (in_params->stats_queue >= max_stats_queues) { 2131 DP_ERR(p_hwfn->cdev, 2132 "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n", 2133 in_params->stats_queue, max_stats_queues); 2134 return NULL; 2135 } 2136 2137 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 2138 if (!qp) { 2139 DP_NOTICE(p_hwfn, "Failed to allocate qed_rdma_qp\n"); 2140 return NULL; 2141 } 2142 2143 rc = qed_roce_alloc_cid(p_hwfn, &qp->icid); 2144 qp->qpid = ((0xFF << 16) | qp->icid); 2145 2146 DP_INFO(p_hwfn, "ROCE qpid=%x\n", qp->qpid); 2147 2148 if (rc) { 2149 kfree(qp); 2150 return NULL; 2151 } 2152 2153 qp->cur_state = QED_ROCE_QP_STATE_RESET; 2154 qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi); 2155 qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo); 2156 qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi); 2157 qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo); 2158 qp->use_srq = in_params->use_srq; 2159 qp->signal_all = in_params->signal_all; 2160 qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey; 2161 qp->pd = in_params->pd; 2162 qp->dpi = in_params->dpi; 2163 qp->sq_cq_id = in_params->sq_cq_id; 2164 qp->sq_num_pages = in_params->sq_num_pages; 2165 qp->sq_pbl_ptr = in_params->sq_pbl_ptr; 2166 qp->rq_cq_id = in_params->rq_cq_id; 2167 qp->rq_num_pages = in_params->rq_num_pages; 2168 qp->rq_pbl_ptr = in_params->rq_pbl_ptr; 2169 qp->srq_id = in_params->srq_id; 2170 qp->req_offloaded = false; 2171 qp->resp_offloaded = false; 2172 qp->e2e_flow_control_en = qp->use_srq ? false : true; 2173 qp->stats_queue = in_params->stats_queue; 2174 2175 out_params->icid = qp->icid; 2176 out_params->qp_id = qp->qpid; 2177 2178 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Create QP, rc = %d\n", rc); 2179 return qp; 2180 } 2181 2182 static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn, 2183 struct qed_rdma_qp *qp, 2184 enum qed_roce_qp_state prev_state, 2185 struct qed_rdma_modify_qp_in_params *params) 2186 { 2187 u32 num_invalidated_mw = 0, num_bound_mw = 0; 2188 int rc = 0; 2189 2190 /* Perform additional operations according to the current state and the 2191 * next state 2192 */ 2193 if (((prev_state == QED_ROCE_QP_STATE_INIT) || 2194 (prev_state == QED_ROCE_QP_STATE_RESET)) && 2195 (qp->cur_state == QED_ROCE_QP_STATE_RTR)) { 2196 /* Init->RTR or Reset->RTR */ 2197 rc = qed_roce_sp_create_responder(p_hwfn, qp); 2198 return rc; 2199 } else if ((prev_state == QED_ROCE_QP_STATE_RTR) && 2200 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2201 /* RTR-> RTS */ 2202 rc = qed_roce_sp_create_requester(p_hwfn, qp); 2203 if (rc) 2204 return rc; 2205 2206 /* Send modify responder ramrod */ 2207 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2208 params->modify_flags); 2209 return rc; 2210 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2211 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2212 /* RTS->RTS */ 2213 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2214 params->modify_flags); 2215 if (rc) 2216 return rc; 2217 2218 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2219 params->modify_flags); 2220 return rc; 2221 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2222 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2223 /* RTS->SQD */ 2224 rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false, 2225 params->modify_flags); 2226 return rc; 2227 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2228 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2229 /* SQD->SQD */ 2230 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2231 params->modify_flags); 2232 if (rc) 2233 return rc; 2234 2235 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2236 params->modify_flags); 2237 return rc; 2238 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2239 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2240 /* SQD->RTS */ 2241 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2242 params->modify_flags); 2243 if (rc) 2244 return rc; 2245 2246 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2247 params->modify_flags); 2248 2249 return rc; 2250 } else if (qp->cur_state == QED_ROCE_QP_STATE_ERR) { 2251 /* ->ERR */ 2252 rc = qed_roce_sp_modify_responder(p_hwfn, qp, true, 2253 params->modify_flags); 2254 if (rc) 2255 return rc; 2256 2257 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true, 2258 params->modify_flags); 2259 return rc; 2260 } else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) { 2261 /* Any state -> RESET */ 2262 u32 cq_prod; 2263 2264 /* Send destroy responder ramrod */ 2265 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, 2266 qp, 2267 &num_invalidated_mw, 2268 &cq_prod); 2269 2270 if (rc) 2271 return rc; 2272 2273 qp->cq_prod = cq_prod; 2274 2275 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, 2276 &num_bound_mw); 2277 2278 if (num_invalidated_mw != num_bound_mw) { 2279 DP_NOTICE(p_hwfn, 2280 "number of invalidate memory windows is different from bounded ones\n"); 2281 return -EINVAL; 2282 } 2283 } else { 2284 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n"); 2285 } 2286 2287 return rc; 2288 } 2289 2290 static int qed_rdma_modify_qp(void *rdma_cxt, 2291 struct qed_rdma_qp *qp, 2292 struct qed_rdma_modify_qp_in_params *params) 2293 { 2294 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2295 enum qed_roce_qp_state prev_state; 2296 int rc = 0; 2297 2298 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x params->new_state=%d\n", 2299 qp->icid, params->new_state); 2300 2301 if (rc) { 2302 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2303 return rc; 2304 } 2305 2306 if (GET_FIELD(params->modify_flags, 2307 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) { 2308 qp->incoming_rdma_read_en = params->incoming_rdma_read_en; 2309 qp->incoming_rdma_write_en = params->incoming_rdma_write_en; 2310 qp->incoming_atomic_en = params->incoming_atomic_en; 2311 } 2312 2313 /* Update QP structure with the updated values */ 2314 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE)) 2315 qp->roce_mode = params->roce_mode; 2316 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)) 2317 qp->pkey = params->pkey; 2318 if (GET_FIELD(params->modify_flags, 2319 QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN)) 2320 qp->e2e_flow_control_en = params->e2e_flow_control_en; 2321 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_DEST_QP)) 2322 qp->dest_qp = params->dest_qp; 2323 if (GET_FIELD(params->modify_flags, 2324 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) { 2325 /* Indicates that the following parameters have changed: 2326 * Traffic class, flow label, hop limit, source GID, 2327 * destination GID, loopback indicator 2328 */ 2329 qp->traffic_class_tos = params->traffic_class_tos; 2330 qp->flow_label = params->flow_label; 2331 qp->hop_limit_ttl = params->hop_limit_ttl; 2332 2333 qp->sgid = params->sgid; 2334 qp->dgid = params->dgid; 2335 qp->udp_src_port = 0; 2336 qp->vlan_id = params->vlan_id; 2337 qp->mtu = params->mtu; 2338 qp->lb_indication = params->lb_indication; 2339 memcpy((u8 *)&qp->remote_mac_addr[0], 2340 (u8 *)¶ms->remote_mac_addr[0], ETH_ALEN); 2341 if (params->use_local_mac) { 2342 memcpy((u8 *)&qp->local_mac_addr[0], 2343 (u8 *)¶ms->local_mac_addr[0], ETH_ALEN); 2344 } else { 2345 memcpy((u8 *)&qp->local_mac_addr[0], 2346 (u8 *)&p_hwfn->hw_info.hw_mac_addr, ETH_ALEN); 2347 } 2348 } 2349 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RQ_PSN)) 2350 qp->rq_psn = params->rq_psn; 2351 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_SQ_PSN)) 2352 qp->sq_psn = params->sq_psn; 2353 if (GET_FIELD(params->modify_flags, 2354 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)) 2355 qp->max_rd_atomic_req = params->max_rd_atomic_req; 2356 if (GET_FIELD(params->modify_flags, 2357 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)) 2358 qp->max_rd_atomic_resp = params->max_rd_atomic_resp; 2359 if (GET_FIELD(params->modify_flags, 2360 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)) 2361 qp->ack_timeout = params->ack_timeout; 2362 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)) 2363 qp->retry_cnt = params->retry_cnt; 2364 if (GET_FIELD(params->modify_flags, 2365 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)) 2366 qp->rnr_retry_cnt = params->rnr_retry_cnt; 2367 if (GET_FIELD(params->modify_flags, 2368 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)) 2369 qp->min_rnr_nak_timer = params->min_rnr_nak_timer; 2370 2371 qp->sqd_async = params->sqd_async; 2372 2373 prev_state = qp->cur_state; 2374 if (GET_FIELD(params->modify_flags, 2375 QED_RDMA_MODIFY_QP_VALID_NEW_STATE)) { 2376 qp->cur_state = params->new_state; 2377 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "qp->cur_state=%d\n", 2378 qp->cur_state); 2379 } 2380 2381 rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params); 2382 2383 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify QP, rc = %d\n", rc); 2384 return rc; 2385 } 2386 2387 static int 2388 qed_rdma_register_tid(void *rdma_cxt, 2389 struct qed_rdma_register_tid_in_params *params) 2390 { 2391 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2392 struct rdma_register_tid_ramrod_data *p_ramrod; 2393 struct qed_sp_init_data init_data; 2394 struct qed_spq_entry *p_ent; 2395 enum rdma_tid_type tid_type; 2396 u8 fw_return_code; 2397 int rc; 2398 2399 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", params->itid); 2400 2401 /* Get SPQ entry */ 2402 memset(&init_data, 0, sizeof(init_data)); 2403 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2404 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2405 2406 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR, 2407 p_hwfn->p_rdma_info->proto, &init_data); 2408 if (rc) { 2409 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2410 return rc; 2411 } 2412 2413 if (p_hwfn->p_rdma_info->last_tid < params->itid) 2414 p_hwfn->p_rdma_info->last_tid = params->itid; 2415 2416 p_ramrod = &p_ent->ramrod.rdma_register_tid; 2417 2418 p_ramrod->flags = 0; 2419 SET_FIELD(p_ramrod->flags, 2420 RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL, 2421 params->pbl_two_level); 2422 2423 SET_FIELD(p_ramrod->flags, 2424 RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, params->zbva); 2425 2426 SET_FIELD(p_ramrod->flags, 2427 RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr); 2428 2429 /* Don't initialize D/C field, as it may override other bits. */ 2430 if (!(params->tid_type == QED_RDMA_TID_FMR) && !(params->dma_mr)) 2431 SET_FIELD(p_ramrod->flags, 2432 RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG, 2433 params->page_size_log - 12); 2434 2435 SET_FIELD(p_ramrod->flags, 2436 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ, 2437 params->remote_read); 2438 2439 SET_FIELD(p_ramrod->flags, 2440 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE, 2441 params->remote_write); 2442 2443 SET_FIELD(p_ramrod->flags, 2444 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC, 2445 params->remote_atomic); 2446 2447 SET_FIELD(p_ramrod->flags, 2448 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE, 2449 params->local_write); 2450 2451 SET_FIELD(p_ramrod->flags, 2452 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, params->local_read); 2453 2454 SET_FIELD(p_ramrod->flags, 2455 RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND, 2456 params->mw_bind); 2457 2458 SET_FIELD(p_ramrod->flags1, 2459 RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG, 2460 params->pbl_page_size_log - 12); 2461 2462 SET_FIELD(p_ramrod->flags2, 2463 RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, params->dma_mr); 2464 2465 switch (params->tid_type) { 2466 case QED_RDMA_TID_REGISTERED_MR: 2467 tid_type = RDMA_TID_REGISTERED_MR; 2468 break; 2469 case QED_RDMA_TID_FMR: 2470 tid_type = RDMA_TID_FMR; 2471 break; 2472 case QED_RDMA_TID_MW_TYPE1: 2473 tid_type = RDMA_TID_MW_TYPE1; 2474 break; 2475 case QED_RDMA_TID_MW_TYPE2A: 2476 tid_type = RDMA_TID_MW_TYPE2A; 2477 break; 2478 default: 2479 rc = -EINVAL; 2480 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2481 return rc; 2482 } 2483 SET_FIELD(p_ramrod->flags1, 2484 RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, tid_type); 2485 2486 p_ramrod->itid = cpu_to_le32(params->itid); 2487 p_ramrod->key = params->key; 2488 p_ramrod->pd = cpu_to_le16(params->pd); 2489 p_ramrod->length_hi = (u8)(params->length >> 32); 2490 p_ramrod->length_lo = DMA_LO_LE(params->length); 2491 if (params->zbva) { 2492 /* Lower 32 bits of the registered MR address. 2493 * In case of zero based MR, will hold FBO 2494 */ 2495 p_ramrod->va.hi = 0; 2496 p_ramrod->va.lo = cpu_to_le32(params->fbo); 2497 } else { 2498 DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); 2499 } 2500 DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr); 2501 2502 /* DIF */ 2503 if (params->dif_enabled) { 2504 SET_FIELD(p_ramrod->flags2, 2505 RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1); 2506 DMA_REGPAIR_LE(p_ramrod->dif_error_addr, 2507 params->dif_error_addr); 2508 DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr); 2509 } 2510 2511 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2512 if (rc) 2513 return rc; 2514 2515 if (fw_return_code != RDMA_RETURN_OK) { 2516 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2517 return -EINVAL; 2518 } 2519 2520 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Register TID, rc = %d\n", rc); 2521 return rc; 2522 } 2523 2524 static int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid) 2525 { 2526 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2527 struct rdma_deregister_tid_ramrod_data *p_ramrod; 2528 struct qed_sp_init_data init_data; 2529 struct qed_spq_entry *p_ent; 2530 struct qed_ptt *p_ptt; 2531 u8 fw_return_code; 2532 int rc; 2533 2534 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 2535 2536 /* Get SPQ entry */ 2537 memset(&init_data, 0, sizeof(init_data)); 2538 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2539 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2540 2541 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_DEREGISTER_MR, 2542 p_hwfn->p_rdma_info->proto, &init_data); 2543 if (rc) { 2544 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2545 return rc; 2546 } 2547 2548 p_ramrod = &p_ent->ramrod.rdma_deregister_tid; 2549 p_ramrod->itid = cpu_to_le32(itid); 2550 2551 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2552 if (rc) { 2553 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2554 return rc; 2555 } 2556 2557 if (fw_return_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) { 2558 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2559 return -EINVAL; 2560 } else if (fw_return_code == RDMA_RETURN_NIG_DRAIN_REQ) { 2561 /* Bit indicating that the TID is in use and a nig drain is 2562 * required before sending the ramrod again 2563 */ 2564 p_ptt = qed_ptt_acquire(p_hwfn); 2565 if (!p_ptt) { 2566 rc = -EBUSY; 2567 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2568 "Failed to acquire PTT\n"); 2569 return rc; 2570 } 2571 2572 rc = qed_mcp_drain(p_hwfn, p_ptt); 2573 if (rc) { 2574 qed_ptt_release(p_hwfn, p_ptt); 2575 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2576 "Drain failed\n"); 2577 return rc; 2578 } 2579 2580 qed_ptt_release(p_hwfn, p_ptt); 2581 2582 /* Resend the ramrod */ 2583 rc = qed_sp_init_request(p_hwfn, &p_ent, 2584 RDMA_RAMROD_DEREGISTER_MR, 2585 p_hwfn->p_rdma_info->proto, 2586 &init_data); 2587 if (rc) { 2588 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2589 "Failed to init sp-element\n"); 2590 return rc; 2591 } 2592 2593 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2594 if (rc) { 2595 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2596 "Ramrod failed\n"); 2597 return rc; 2598 } 2599 2600 if (fw_return_code != RDMA_RETURN_OK) { 2601 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", 2602 fw_return_code); 2603 return rc; 2604 } 2605 } 2606 2607 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "De-registered TID, rc = %d\n", rc); 2608 return rc; 2609 } 2610 2611 static void qed_roce_free_real_icid(struct qed_hwfn *p_hwfn, u16 icid) 2612 { 2613 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 2614 u32 start_cid, cid, xcid; 2615 2616 /* an even icid belongs to a responder while an odd icid belongs to a 2617 * requester. The 'cid' received as an input can be either. We calculate 2618 * the "partner" icid and call it xcid. Only if both are free then the 2619 * "cid" map can be cleared. 2620 */ 2621 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, p_rdma_info->proto); 2622 cid = icid - start_cid; 2623 xcid = cid ^ 1; 2624 2625 spin_lock_bh(&p_rdma_info->lock); 2626 2627 qed_bmap_release_id(p_hwfn, &p_rdma_info->real_cid_map, cid); 2628 if (qed_bmap_test_id(p_hwfn, &p_rdma_info->real_cid_map, xcid) == 0) { 2629 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, cid); 2630 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, xcid); 2631 } 2632 2633 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 2634 } 2635 2636 static void *qed_rdma_get_rdma_ctx(struct qed_dev *cdev) 2637 { 2638 return QED_LEADING_HWFN(cdev); 2639 } 2640 2641 static void qed_rdma_dpm_conf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2642 { 2643 u32 val; 2644 2645 val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1; 2646 2647 qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val); 2648 DP_VERBOSE(p_hwfn, (QED_MSG_DCB | QED_MSG_RDMA), 2649 "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n", 2650 val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm); 2651 } 2652 2653 void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2654 { 2655 p_hwfn->db_bar_no_edpm = true; 2656 2657 qed_rdma_dpm_conf(p_hwfn, p_ptt); 2658 } 2659 2660 static int qed_rdma_start(void *rdma_cxt, 2661 struct qed_rdma_start_in_params *params) 2662 { 2663 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2664 struct qed_ptt *p_ptt; 2665 int rc = -EBUSY; 2666 2667 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2668 "desired_cnq = %08x\n", params->desired_cnq); 2669 2670 p_ptt = qed_ptt_acquire(p_hwfn); 2671 if (!p_ptt) 2672 goto err; 2673 2674 rc = qed_rdma_alloc(p_hwfn, p_ptt, params); 2675 if (rc) 2676 goto err1; 2677 2678 rc = qed_rdma_setup(p_hwfn, p_ptt, params); 2679 if (rc) 2680 goto err2; 2681 2682 qed_ptt_release(p_hwfn, p_ptt); 2683 2684 return rc; 2685 2686 err2: 2687 qed_rdma_free(p_hwfn); 2688 err1: 2689 qed_ptt_release(p_hwfn, p_ptt); 2690 err: 2691 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA start - error, rc = %d\n", rc); 2692 return rc; 2693 } 2694 2695 static int qed_rdma_init(struct qed_dev *cdev, 2696 struct qed_rdma_start_in_params *params) 2697 { 2698 return qed_rdma_start(QED_LEADING_HWFN(cdev), params); 2699 } 2700 2701 static void qed_rdma_remove_user(void *rdma_cxt, u16 dpi) 2702 { 2703 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2704 2705 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "dpi = %08x\n", dpi); 2706 2707 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 2708 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi); 2709 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 2710 } 2711 2712 void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2713 u8 connection_handle, 2714 void *cookie, 2715 dma_addr_t first_frag_addr, 2716 bool b_last_fragment, bool b_last_packet) 2717 { 2718 struct qed_roce_ll2_packet *packet = cookie; 2719 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2720 2721 roce_ll2->cbs.tx_cb(roce_ll2->cb_cookie, packet); 2722 } 2723 2724 void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2725 u8 connection_handle, 2726 void *cookie, 2727 dma_addr_t first_frag_addr, 2728 bool b_last_fragment, bool b_last_packet) 2729 { 2730 qed_ll2b_complete_tx_gsi_packet(p_hwfn, connection_handle, 2731 cookie, first_frag_addr, 2732 b_last_fragment, b_last_packet); 2733 } 2734 2735 void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, 2736 u8 connection_handle, 2737 void *cookie, 2738 dma_addr_t rx_buf_addr, 2739 u16 data_length, 2740 u8 data_length_error, 2741 u16 parse_flags, 2742 u16 vlan, 2743 u32 src_mac_addr_hi, 2744 u16 src_mac_addr_lo, bool b_last_packet) 2745 { 2746 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2747 struct qed_roce_ll2_rx_params params; 2748 struct qed_dev *cdev = p_hwfn->cdev; 2749 struct qed_roce_ll2_packet pkt; 2750 2751 DP_VERBOSE(cdev, 2752 QED_MSG_LL2, 2753 "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n", 2754 (void *)(uintptr_t)rx_buf_addr, 2755 data_length, data_length_error); 2756 2757 memset(&pkt, 0, sizeof(pkt)); 2758 pkt.n_seg = 1; 2759 pkt.payload[0].baddr = rx_buf_addr; 2760 pkt.payload[0].len = data_length; 2761 2762 memset(¶ms, 0, sizeof(params)); 2763 params.vlan_id = vlan; 2764 *((u32 *)¶ms.smac[0]) = ntohl(src_mac_addr_hi); 2765 *((u16 *)¶ms.smac[4]) = ntohs(src_mac_addr_lo); 2766 2767 if (data_length_error) { 2768 DP_ERR(cdev, 2769 "roce ll2 rx complete: data length error %d, length=%d\n", 2770 data_length_error, data_length); 2771 params.rc = -EINVAL; 2772 } 2773 2774 roce_ll2->cbs.rx_cb(roce_ll2->cb_cookie, &pkt, ¶ms); 2775 } 2776 2777 static int qed_roce_ll2_set_mac_filter(struct qed_dev *cdev, 2778 u8 *old_mac_address, 2779 u8 *new_mac_address) 2780 { 2781 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2782 struct qed_ptt *p_ptt; 2783 int rc = 0; 2784 2785 if (!hwfn->ll2 || hwfn->ll2->handle == QED_LL2_UNUSED_HANDLE) { 2786 DP_ERR(cdev, 2787 "qed roce mac filter failed - roce_info/ll2 NULL\n"); 2788 return -EINVAL; 2789 } 2790 2791 p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev)); 2792 if (!p_ptt) { 2793 DP_ERR(cdev, 2794 "qed roce ll2 mac filter set: failed to acquire PTT\n"); 2795 return -EINVAL; 2796 } 2797 2798 mutex_lock(&hwfn->ll2->lock); 2799 if (old_mac_address) 2800 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2801 old_mac_address); 2802 if (new_mac_address) 2803 rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2804 new_mac_address); 2805 mutex_unlock(&hwfn->ll2->lock); 2806 2807 qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt); 2808 2809 if (rc) 2810 DP_ERR(cdev, 2811 "qed roce ll2 mac filter set: failed to add mac filter\n"); 2812 2813 return rc; 2814 } 2815 2816 static int qed_roce_ll2_start(struct qed_dev *cdev, 2817 struct qed_roce_ll2_params *params) 2818 { 2819 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2820 struct qed_roce_ll2_info *roce_ll2; 2821 struct qed_ll2_conn ll2_params; 2822 int rc; 2823 2824 if (!params) { 2825 DP_ERR(cdev, "qed roce ll2 start: failed due to NULL params\n"); 2826 return -EINVAL; 2827 } 2828 if (!params->cbs.tx_cb || !params->cbs.rx_cb) { 2829 DP_ERR(cdev, 2830 "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n", 2831 params->cbs.tx_cb, params->cbs.rx_cb); 2832 return -EINVAL; 2833 } 2834 if (!is_valid_ether_addr(params->mac_address)) { 2835 DP_ERR(cdev, 2836 "qed roce ll2 start: failed due to invalid Ethernet address %pM\n", 2837 params->mac_address); 2838 return -EINVAL; 2839 } 2840 2841 /* Initialize */ 2842 roce_ll2 = kzalloc(sizeof(*roce_ll2), GFP_ATOMIC); 2843 if (!roce_ll2) { 2844 DP_ERR(cdev, "qed roce ll2 start: failed memory allocation\n"); 2845 return -ENOMEM; 2846 } 2847 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2848 roce_ll2->cbs = params->cbs; 2849 roce_ll2->cb_cookie = params->cb_cookie; 2850 mutex_init(&roce_ll2->lock); 2851 2852 memset(&ll2_params, 0, sizeof(ll2_params)); 2853 ll2_params.conn_type = QED_LL2_TYPE_ROCE; 2854 ll2_params.mtu = params->mtu; 2855 ll2_params.rx_drop_ttl0_flg = true; 2856 ll2_params.rx_vlan_removal_en = false; 2857 ll2_params.tx_dest = CORE_TX_DEST_NW; 2858 ll2_params.ai_err_packet_too_big = LL2_DROP_PACKET; 2859 ll2_params.ai_err_no_buf = LL2_DROP_PACKET; 2860 ll2_params.gsi_enable = true; 2861 2862 rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_params, 2863 params->max_rx_buffers, 2864 params->max_tx_buffers, 2865 &roce_ll2->handle); 2866 if (rc) { 2867 DP_ERR(cdev, 2868 "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n", 2869 rc); 2870 goto err; 2871 } 2872 2873 rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev), 2874 roce_ll2->handle); 2875 if (rc) { 2876 DP_ERR(cdev, 2877 "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n", 2878 rc); 2879 goto err1; 2880 } 2881 2882 hwfn->ll2 = roce_ll2; 2883 2884 rc = qed_roce_ll2_set_mac_filter(cdev, NULL, params->mac_address); 2885 if (rc) { 2886 hwfn->ll2 = NULL; 2887 goto err2; 2888 } 2889 ether_addr_copy(roce_ll2->mac_address, params->mac_address); 2890 2891 return 0; 2892 2893 err2: 2894 qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2895 err1: 2896 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2897 err: 2898 kfree(roce_ll2); 2899 return rc; 2900 } 2901 2902 static int qed_roce_ll2_stop(struct qed_dev *cdev) 2903 { 2904 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2905 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2906 int rc; 2907 2908 if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) { 2909 DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n"); 2910 return -EINVAL; 2911 } 2912 2913 /* remove LL2 MAC address filter */ 2914 rc = qed_roce_ll2_set_mac_filter(cdev, roce_ll2->mac_address, NULL); 2915 eth_zero_addr(roce_ll2->mac_address); 2916 2917 rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), 2918 roce_ll2->handle); 2919 if (rc) 2920 DP_ERR(cdev, 2921 "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n", 2922 rc); 2923 2924 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2925 2926 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2927 2928 kfree(roce_ll2); 2929 2930 return rc; 2931 } 2932 2933 static int qed_roce_ll2_tx(struct qed_dev *cdev, 2934 struct qed_roce_ll2_packet *pkt, 2935 struct qed_roce_ll2_tx_params *params) 2936 { 2937 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2938 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2939 enum qed_ll2_roce_flavor_type qed_roce_flavor; 2940 struct qed_ll2_tx_pkt_info ll2_pkt; 2941 u8 flags = 0; 2942 int rc; 2943 int i; 2944 2945 if (!pkt || !params) { 2946 DP_ERR(cdev, 2947 "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n", 2948 cdev, pkt, params); 2949 return -EINVAL; 2950 } 2951 2952 qed_roce_flavor = (pkt->roce_mode == ROCE_V1) ? QED_LL2_ROCE 2953 : QED_LL2_RROCE; 2954 2955 if (pkt->roce_mode == ROCE_V2_IPV4) 2956 flags |= BIT(CORE_TX_BD_DATA_IP_CSUM_SHIFT); 2957 2958 /* Tx header */ 2959 memset(&ll2_pkt, 0, sizeof(ll2_pkt)); 2960 ll2_pkt.num_of_bds = 1 + pkt->n_seg; 2961 ll2_pkt.bd_flags = flags; 2962 ll2_pkt.tx_dest = QED_LL2_TX_DEST_NW; 2963 ll2_pkt.qed_roce_flavor = qed_roce_flavor; 2964 ll2_pkt.first_frag = pkt->header.baddr; 2965 ll2_pkt.first_frag_len = pkt->header.len; 2966 ll2_pkt.cookie = pkt; 2967 2968 rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev), 2969 roce_ll2->handle, 2970 &ll2_pkt, 1); 2971 if (rc) { 2972 DP_ERR(cdev, "roce ll2 tx: header failed (rc=%d)\n", rc); 2973 return QED_ROCE_TX_HEAD_FAILURE; 2974 } 2975 2976 /* Tx payload */ 2977 for (i = 0; i < pkt->n_seg; i++) { 2978 rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev), 2979 roce_ll2->handle, 2980 pkt->payload[i].baddr, 2981 pkt->payload[i].len); 2982 if (rc) { 2983 /* If failed not much to do here, partial packet has 2984 * been posted * we can't free memory, will need to wait 2985 * for completion 2986 */ 2987 DP_ERR(cdev, 2988 "roce ll2 tx: payload failed (rc=%d)\n", rc); 2989 return QED_ROCE_TX_FRAG_FAILURE; 2990 } 2991 } 2992 2993 return 0; 2994 } 2995 2996 static int qed_roce_ll2_post_rx_buffer(struct qed_dev *cdev, 2997 struct qed_roce_ll2_buffer *buf, 2998 u64 cookie, u8 notify_fw) 2999 { 3000 return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev), 3001 QED_LEADING_HWFN(cdev)->ll2->handle, 3002 buf->baddr, buf->len, 3003 (void *)(uintptr_t)cookie, notify_fw); 3004 } 3005 3006 static int qed_roce_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats) 3007 { 3008 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 3009 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 3010 3011 return qed_ll2_get_stats(QED_LEADING_HWFN(cdev), 3012 roce_ll2->handle, stats); 3013 } 3014 3015 static const struct qed_rdma_ops qed_rdma_ops_pass = { 3016 .common = &qed_common_ops_pass, 3017 .fill_dev_info = &qed_fill_rdma_dev_info, 3018 .rdma_get_rdma_ctx = &qed_rdma_get_rdma_ctx, 3019 .rdma_init = &qed_rdma_init, 3020 .rdma_add_user = &qed_rdma_add_user, 3021 .rdma_remove_user = &qed_rdma_remove_user, 3022 .rdma_stop = &qed_rdma_stop, 3023 .rdma_query_port = &qed_rdma_query_port, 3024 .rdma_query_device = &qed_rdma_query_device, 3025 .rdma_get_start_sb = &qed_rdma_get_sb_start, 3026 .rdma_get_rdma_int = &qed_rdma_get_int, 3027 .rdma_set_rdma_int = &qed_rdma_set_int, 3028 .rdma_get_min_cnq_msix = &qed_rdma_get_min_cnq_msix, 3029 .rdma_cnq_prod_update = &qed_rdma_cnq_prod_update, 3030 .rdma_alloc_pd = &qed_rdma_alloc_pd, 3031 .rdma_dealloc_pd = &qed_rdma_free_pd, 3032 .rdma_create_cq = &qed_rdma_create_cq, 3033 .rdma_destroy_cq = &qed_rdma_destroy_cq, 3034 .rdma_create_qp = &qed_rdma_create_qp, 3035 .rdma_modify_qp = &qed_rdma_modify_qp, 3036 .rdma_query_qp = &qed_rdma_query_qp, 3037 .rdma_destroy_qp = &qed_rdma_destroy_qp, 3038 .rdma_alloc_tid = &qed_rdma_alloc_tid, 3039 .rdma_free_tid = &qed_rdma_free_tid, 3040 .rdma_register_tid = &qed_rdma_register_tid, 3041 .rdma_deregister_tid = &qed_rdma_deregister_tid, 3042 .roce_ll2_start = &qed_roce_ll2_start, 3043 .roce_ll2_stop = &qed_roce_ll2_stop, 3044 .roce_ll2_tx = &qed_roce_ll2_tx, 3045 .roce_ll2_post_rx_buffer = &qed_roce_ll2_post_rx_buffer, 3046 .roce_ll2_set_mac_filter = &qed_roce_ll2_set_mac_filter, 3047 .roce_ll2_stats = &qed_roce_ll2_stats, 3048 }; 3049 3050 const struct qed_rdma_ops *qed_get_rdma_ops(void) 3051 { 3052 return &qed_rdma_ops_pass; 3053 } 3054 EXPORT_SYMBOL(qed_get_rdma_ops); 3055