1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2016 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/types.h> 33 #include <asm/byteorder.h> 34 #include <linux/bitops.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/etherdevice.h> 39 #include <linux/if_ether.h> 40 #include <linux/if_vlan.h> 41 #include <linux/io.h> 42 #include <linux/ip.h> 43 #include <linux/ipv6.h> 44 #include <linux/kernel.h> 45 #include <linux/list.h> 46 #include <linux/module.h> 47 #include <linux/mutex.h> 48 #include <linux/pci.h> 49 #include <linux/slab.h> 50 #include <linux/spinlock.h> 51 #include <linux/string.h> 52 #include <linux/tcp.h> 53 #include <linux/bitops.h> 54 #include <linux/qed/qed_roce_if.h> 55 #include <linux/qed/qed_roce_if.h> 56 #include "qed.h" 57 #include "qed_cxt.h" 58 #include "qed_hsi.h" 59 #include "qed_hw.h" 60 #include "qed_init_ops.h" 61 #include "qed_int.h" 62 #include "qed_ll2.h" 63 #include "qed_mcp.h" 64 #include "qed_reg_addr.h" 65 #include "qed_sp.h" 66 #include "qed_roce.h" 67 #include "qed_ll2.h" 68 69 void qed_async_roce_event(struct qed_hwfn *p_hwfn, 70 struct event_ring_entry *p_eqe) 71 { 72 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 73 74 p_rdma_info->events.affiliated_event(p_rdma_info->events.context, 75 p_eqe->opcode, &p_eqe->data); 76 } 77 78 static int qed_rdma_bmap_alloc(struct qed_hwfn *p_hwfn, 79 struct qed_bmap *bmap, u32 max_count) 80 { 81 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "max_count = %08x\n", max_count); 82 83 bmap->max_count = max_count; 84 85 bmap->bitmap = kzalloc(BITS_TO_LONGS(max_count) * sizeof(long), 86 GFP_KERNEL); 87 if (!bmap->bitmap) { 88 DP_NOTICE(p_hwfn, 89 "qed bmap alloc failed: cannot allocate memory (bitmap)\n"); 90 return -ENOMEM; 91 } 92 93 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocated bitmap %p\n", 94 bmap->bitmap); 95 return 0; 96 } 97 98 static int qed_rdma_bmap_alloc_id(struct qed_hwfn *p_hwfn, 99 struct qed_bmap *bmap, u32 *id_num) 100 { 101 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "bmap = %p\n", bmap); 102 103 *id_num = find_first_zero_bit(bmap->bitmap, bmap->max_count); 104 105 if (*id_num >= bmap->max_count) { 106 DP_NOTICE(p_hwfn, "no id available max_count=%d\n", 107 bmap->max_count); 108 return -EINVAL; 109 } 110 111 __set_bit(*id_num, bmap->bitmap); 112 113 return 0; 114 } 115 116 static void qed_bmap_release_id(struct qed_hwfn *p_hwfn, 117 struct qed_bmap *bmap, u32 id_num) 118 { 119 bool b_acquired; 120 121 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "id_num = %08x", id_num); 122 if (id_num >= bmap->max_count) 123 return; 124 125 b_acquired = test_and_clear_bit(id_num, bmap->bitmap); 126 if (!b_acquired) { 127 DP_NOTICE(p_hwfn, "ID %d already released\n", id_num); 128 return; 129 } 130 } 131 132 static u32 qed_rdma_get_sb_id(void *p_hwfn, u32 rel_sb_id) 133 { 134 /* First sb id for RoCE is after all the l2 sb */ 135 return FEAT_NUM((struct qed_hwfn *)p_hwfn, QED_PF_L2_QUE) + rel_sb_id; 136 } 137 138 static int qed_rdma_alloc(struct qed_hwfn *p_hwfn, 139 struct qed_ptt *p_ptt, 140 struct qed_rdma_start_in_params *params) 141 { 142 struct qed_rdma_info *p_rdma_info; 143 u32 num_cons, num_tasks; 144 int rc = -ENOMEM; 145 146 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocating RDMA\n"); 147 148 /* Allocate a struct with current pf rdma info */ 149 p_rdma_info = kzalloc(sizeof(*p_rdma_info), GFP_KERNEL); 150 if (!p_rdma_info) { 151 DP_NOTICE(p_hwfn, 152 "qed rdma alloc failed: cannot allocate memory (rdma info). rc = %d\n", 153 rc); 154 return rc; 155 } 156 157 p_hwfn->p_rdma_info = p_rdma_info; 158 p_rdma_info->proto = PROTOCOLID_ROCE; 159 160 num_cons = qed_cxt_get_proto_cid_count(p_hwfn, p_rdma_info->proto, 161 NULL); 162 163 p_rdma_info->num_qps = num_cons / 2; 164 165 num_tasks = qed_cxt_get_proto_tid_count(p_hwfn, PROTOCOLID_ROCE); 166 167 /* Each MR uses a single task */ 168 p_rdma_info->num_mrs = num_tasks; 169 170 /* Queue zone lines are shared between RoCE and L2 in such a way that 171 * they can be used by each without obstructing the other. 172 */ 173 p_rdma_info->queue_zone_base = (u16)FEAT_NUM(p_hwfn, QED_L2_QUEUE); 174 175 /* Allocate a struct with device params and fill it */ 176 p_rdma_info->dev = kzalloc(sizeof(*p_rdma_info->dev), GFP_KERNEL); 177 if (!p_rdma_info->dev) { 178 DP_NOTICE(p_hwfn, 179 "qed rdma alloc failed: cannot allocate memory (rdma info dev). rc = %d\n", 180 rc); 181 goto free_rdma_info; 182 } 183 184 /* Allocate a struct with port params and fill it */ 185 p_rdma_info->port = kzalloc(sizeof(*p_rdma_info->port), GFP_KERNEL); 186 if (!p_rdma_info->port) { 187 DP_NOTICE(p_hwfn, 188 "qed rdma alloc failed: cannot allocate memory (rdma info port). rc = %d\n", 189 rc); 190 goto free_rdma_dev; 191 } 192 193 /* Allocate bit map for pd's */ 194 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->pd_map, RDMA_MAX_PDS); 195 if (rc) { 196 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 197 "Failed to allocate pd_map, rc = %d\n", 198 rc); 199 goto free_rdma_port; 200 } 201 202 /* Allocate DPI bitmap */ 203 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->dpi_map, 204 p_hwfn->dpi_count); 205 if (rc) { 206 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 207 "Failed to allocate DPI bitmap, rc = %d\n", rc); 208 goto free_pd_map; 209 } 210 211 /* Allocate bitmap for cq's. The maximum number of CQs is bounded to 212 * twice the number of QPs. 213 */ 214 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cq_map, 215 p_rdma_info->num_qps * 2); 216 if (rc) { 217 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 218 "Failed to allocate cq bitmap, rc = %d\n", rc); 219 goto free_dpi_map; 220 } 221 222 /* Allocate bitmap for toggle bit for cq icids 223 * We toggle the bit every time we create or resize cq for a given icid. 224 * The maximum number of CQs is bounded to twice the number of QPs. 225 */ 226 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->toggle_bits, 227 p_rdma_info->num_qps * 2); 228 if (rc) { 229 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 230 "Failed to allocate toogle bits, rc = %d\n", rc); 231 goto free_cq_map; 232 } 233 234 /* Allocate bitmap for itids */ 235 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->tid_map, 236 p_rdma_info->num_mrs); 237 if (rc) { 238 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 239 "Failed to allocate itids bitmaps, rc = %d\n", rc); 240 goto free_toggle_map; 241 } 242 243 /* Allocate bitmap for cids used for qps. */ 244 rc = qed_rdma_bmap_alloc(p_hwfn, &p_rdma_info->cid_map, num_cons); 245 if (rc) { 246 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 247 "Failed to allocate cid bitmap, rc = %d\n", rc); 248 goto free_tid_map; 249 } 250 251 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocation successful\n"); 252 return 0; 253 254 free_tid_map: 255 kfree(p_rdma_info->tid_map.bitmap); 256 free_toggle_map: 257 kfree(p_rdma_info->toggle_bits.bitmap); 258 free_cq_map: 259 kfree(p_rdma_info->cq_map.bitmap); 260 free_dpi_map: 261 kfree(p_rdma_info->dpi_map.bitmap); 262 free_pd_map: 263 kfree(p_rdma_info->pd_map.bitmap); 264 free_rdma_port: 265 kfree(p_rdma_info->port); 266 free_rdma_dev: 267 kfree(p_rdma_info->dev); 268 free_rdma_info: 269 kfree(p_rdma_info); 270 271 return rc; 272 } 273 274 static void qed_rdma_resc_free(struct qed_hwfn *p_hwfn) 275 { 276 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 277 278 kfree(p_rdma_info->cid_map.bitmap); 279 kfree(p_rdma_info->tid_map.bitmap); 280 kfree(p_rdma_info->toggle_bits.bitmap); 281 kfree(p_rdma_info->cq_map.bitmap); 282 kfree(p_rdma_info->dpi_map.bitmap); 283 kfree(p_rdma_info->pd_map.bitmap); 284 285 kfree(p_rdma_info->port); 286 kfree(p_rdma_info->dev); 287 288 kfree(p_rdma_info); 289 } 290 291 static void qed_rdma_free(struct qed_hwfn *p_hwfn) 292 { 293 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Freeing RDMA\n"); 294 295 qed_rdma_resc_free(p_hwfn); 296 } 297 298 static void qed_rdma_get_guid(struct qed_hwfn *p_hwfn, u8 *guid) 299 { 300 guid[0] = p_hwfn->hw_info.hw_mac_addr[0] ^ 2; 301 guid[1] = p_hwfn->hw_info.hw_mac_addr[1]; 302 guid[2] = p_hwfn->hw_info.hw_mac_addr[2]; 303 guid[3] = 0xff; 304 guid[4] = 0xfe; 305 guid[5] = p_hwfn->hw_info.hw_mac_addr[3]; 306 guid[6] = p_hwfn->hw_info.hw_mac_addr[4]; 307 guid[7] = p_hwfn->hw_info.hw_mac_addr[5]; 308 } 309 310 static void qed_rdma_init_events(struct qed_hwfn *p_hwfn, 311 struct qed_rdma_start_in_params *params) 312 { 313 struct qed_rdma_events *events; 314 315 events = &p_hwfn->p_rdma_info->events; 316 317 events->unaffiliated_event = params->events->unaffiliated_event; 318 events->affiliated_event = params->events->affiliated_event; 319 events->context = params->events->context; 320 } 321 322 static void qed_rdma_init_devinfo(struct qed_hwfn *p_hwfn, 323 struct qed_rdma_start_in_params *params) 324 { 325 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 326 struct qed_dev *cdev = p_hwfn->cdev; 327 u32 pci_status_control; 328 u32 num_qps; 329 330 /* Vendor specific information */ 331 dev->vendor_id = cdev->vendor_id; 332 dev->vendor_part_id = cdev->device_id; 333 dev->hw_ver = 0; 334 dev->fw_ver = (FW_MAJOR_VERSION << 24) | (FW_MINOR_VERSION << 16) | 335 (FW_REVISION_VERSION << 8) | (FW_ENGINEERING_VERSION); 336 337 qed_rdma_get_guid(p_hwfn, (u8 *)&dev->sys_image_guid); 338 dev->node_guid = dev->sys_image_guid; 339 340 dev->max_sge = min_t(u32, RDMA_MAX_SGE_PER_SQ_WQE, 341 RDMA_MAX_SGE_PER_RQ_WQE); 342 343 if (cdev->rdma_max_sge) 344 dev->max_sge = min_t(u32, cdev->rdma_max_sge, dev->max_sge); 345 346 dev->max_inline = ROCE_REQ_MAX_INLINE_DATA_SIZE; 347 348 dev->max_inline = (cdev->rdma_max_inline) ? 349 min_t(u32, cdev->rdma_max_inline, dev->max_inline) : 350 dev->max_inline; 351 352 dev->max_wqe = QED_RDMA_MAX_WQE; 353 dev->max_cnq = (u8)FEAT_NUM(p_hwfn, QED_RDMA_CNQ); 354 355 /* The number of QPs may be higher than QED_ROCE_MAX_QPS, because 356 * it is up-aligned to 16 and then to ILT page size within qed cxt. 357 * This is OK in terms of ILT but we don't want to configure the FW 358 * above its abilities 359 */ 360 num_qps = ROCE_MAX_QPS; 361 num_qps = min_t(u64, num_qps, p_hwfn->p_rdma_info->num_qps); 362 dev->max_qp = num_qps; 363 364 /* CQs uses the same icids that QPs use hence they are limited by the 365 * number of icids. There are two icids per QP. 366 */ 367 dev->max_cq = num_qps * 2; 368 369 /* The number of mrs is smaller by 1 since the first is reserved */ 370 dev->max_mr = p_hwfn->p_rdma_info->num_mrs - 1; 371 dev->max_mr_size = QED_RDMA_MAX_MR_SIZE; 372 373 /* The maximum CQE capacity per CQ supported. 374 * max number of cqes will be in two layer pbl, 375 * 8 is the pointer size in bytes 376 * 32 is the size of cq element in bytes 377 */ 378 if (params->cq_mode == QED_RDMA_CQ_MODE_32_BITS) 379 dev->max_cqe = QED_RDMA_MAX_CQE_32_BIT; 380 else 381 dev->max_cqe = QED_RDMA_MAX_CQE_16_BIT; 382 383 dev->max_mw = 0; 384 dev->max_fmr = QED_RDMA_MAX_FMR; 385 dev->max_mr_mw_fmr_pbl = (PAGE_SIZE / 8) * (PAGE_SIZE / 8); 386 dev->max_mr_mw_fmr_size = dev->max_mr_mw_fmr_pbl * PAGE_SIZE; 387 dev->max_pkey = QED_RDMA_MAX_P_KEY; 388 389 dev->max_qp_resp_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 390 (RDMA_RESP_RD_ATOMIC_ELM_SIZE * 2); 391 dev->max_qp_req_rd_atomic_resc = RDMA_RING_PAGE_SIZE / 392 RDMA_REQ_RD_ATOMIC_ELM_SIZE; 393 dev->max_dev_resp_rd_atomic_resc = dev->max_qp_resp_rd_atomic_resc * 394 p_hwfn->p_rdma_info->num_qps; 395 dev->page_size_caps = QED_RDMA_PAGE_SIZE_CAPS; 396 dev->dev_ack_delay = QED_RDMA_ACK_DELAY; 397 dev->max_pd = RDMA_MAX_PDS; 398 dev->max_ah = p_hwfn->p_rdma_info->num_qps; 399 dev->max_stats_queues = (u8)RESC_NUM(p_hwfn, QED_RDMA_STATS_QUEUE); 400 401 /* Set capablities */ 402 dev->dev_caps = 0; 403 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RNR_NAK, 1); 404 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_ACTIVE_EVENT, 1); 405 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_PORT_CHANGE_EVENT, 1); 406 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_RESIZE_CQ, 1); 407 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_MEMORY_EXT, 1); 408 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_BASE_QUEUE_EXT, 1); 409 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ZBVA, 1); 410 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_LOCAL_INV_FENCE, 1); 411 412 /* Check atomic operations support in PCI configuration space. */ 413 pci_read_config_dword(cdev->pdev, 414 cdev->pdev->pcie_cap + PCI_EXP_DEVCTL2, 415 &pci_status_control); 416 417 if (pci_status_control & PCI_EXP_DEVCTL2_LTR_EN) 418 SET_FIELD(dev->dev_caps, QED_RDMA_DEV_CAP_ATOMIC_OP, 1); 419 } 420 421 static void qed_rdma_init_port(struct qed_hwfn *p_hwfn) 422 { 423 struct qed_rdma_port *port = p_hwfn->p_rdma_info->port; 424 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 425 426 port->port_state = p_hwfn->mcp_info->link_output.link_up ? 427 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 428 429 port->max_msg_size = min_t(u64, 430 (dev->max_mr_mw_fmr_size * 431 p_hwfn->cdev->rdma_max_sge), 432 BIT(31)); 433 434 port->pkey_bad_counter = 0; 435 } 436 437 static int qed_rdma_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 438 { 439 u32 ll2_ethertype_en; 440 441 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW\n"); 442 p_hwfn->b_rdma_enabled_in_prs = false; 443 444 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 445 446 p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_ROCE; 447 448 /* We delay writing to this reg until first cid is allocated. See 449 * qed_cxt_dynamic_ilt_alloc function for more details 450 */ 451 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 452 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 453 (ll2_ethertype_en | 0x01)); 454 455 if (qed_cxt_get_proto_cid_start(p_hwfn, PROTOCOLID_ROCE) % 2) { 456 DP_NOTICE(p_hwfn, "The first RoCE's cid should be even\n"); 457 return -EINVAL; 458 } 459 460 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Initializing HW - Done\n"); 461 return 0; 462 } 463 464 static int qed_rdma_start_fw(struct qed_hwfn *p_hwfn, 465 struct qed_rdma_start_in_params *params, 466 struct qed_ptt *p_ptt) 467 { 468 struct rdma_init_func_ramrod_data *p_ramrod; 469 struct qed_rdma_cnq_params *p_cnq_pbl_list; 470 struct rdma_init_func_hdr *p_params_header; 471 struct rdma_cnq_params *p_cnq_params; 472 struct qed_sp_init_data init_data; 473 struct qed_spq_entry *p_ent; 474 u32 cnq_id, sb_id; 475 int rc; 476 477 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Starting FW\n"); 478 479 /* Save the number of cnqs for the function close ramrod */ 480 p_hwfn->p_rdma_info->num_cnqs = params->desired_cnq; 481 482 /* Get SPQ entry */ 483 memset(&init_data, 0, sizeof(init_data)); 484 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 485 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 486 487 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_INIT, 488 p_hwfn->p_rdma_info->proto, &init_data); 489 if (rc) 490 return rc; 491 492 p_ramrod = &p_ent->ramrod.roce_init_func.rdma; 493 494 p_params_header = &p_ramrod->params_header; 495 p_params_header->cnq_start_offset = (u8)RESC_START(p_hwfn, 496 QED_RDMA_CNQ_RAM); 497 p_params_header->num_cnqs = params->desired_cnq; 498 499 if (params->cq_mode == QED_RDMA_CQ_MODE_16_BITS) 500 p_params_header->cq_ring_mode = 1; 501 else 502 p_params_header->cq_ring_mode = 0; 503 504 for (cnq_id = 0; cnq_id < params->desired_cnq; cnq_id++) { 505 sb_id = qed_rdma_get_sb_id(p_hwfn, cnq_id); 506 p_cnq_params = &p_ramrod->cnq_params[cnq_id]; 507 p_cnq_pbl_list = ¶ms->cnq_pbl_list[cnq_id]; 508 p_cnq_params->sb_num = 509 cpu_to_le16(p_hwfn->sbs_info[sb_id]->igu_sb_id); 510 511 p_cnq_params->sb_index = p_hwfn->pf_params.rdma_pf_params.gl_pi; 512 p_cnq_params->num_pbl_pages = p_cnq_pbl_list->num_pbl_pages; 513 514 DMA_REGPAIR_LE(p_cnq_params->pbl_base_addr, 515 p_cnq_pbl_list->pbl_ptr); 516 517 /* we assume here that cnq_id and qz_offset are the same */ 518 p_cnq_params->queue_zone_num = 519 cpu_to_le16(p_hwfn->p_rdma_info->queue_zone_base + 520 cnq_id); 521 } 522 523 return qed_spq_post(p_hwfn, p_ent, NULL); 524 } 525 526 static int qed_rdma_alloc_tid(void *rdma_cxt, u32 *itid) 527 { 528 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 529 int rc; 530 531 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID\n"); 532 533 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 534 rc = qed_rdma_bmap_alloc_id(p_hwfn, 535 &p_hwfn->p_rdma_info->tid_map, itid); 536 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 537 if (rc) 538 goto out; 539 540 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_TASK, *itid); 541 out: 542 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Allocate TID - done, rc = %d\n", rc); 543 return rc; 544 } 545 546 static int qed_rdma_reserve_lkey(struct qed_hwfn *p_hwfn) 547 { 548 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 549 550 /* The first DPI is reserved for the Kernel */ 551 __set_bit(0, p_hwfn->p_rdma_info->dpi_map.bitmap); 552 553 /* Tid 0 will be used as the key for "reserved MR". 554 * The driver should allocate memory for it so it can be loaded but no 555 * ramrod should be passed on it. 556 */ 557 qed_rdma_alloc_tid(p_hwfn, &dev->reserved_lkey); 558 if (dev->reserved_lkey != RDMA_RESERVED_LKEY) { 559 DP_NOTICE(p_hwfn, 560 "Reserved lkey should be equal to RDMA_RESERVED_LKEY\n"); 561 return -EINVAL; 562 } 563 564 return 0; 565 } 566 567 static int qed_rdma_setup(struct qed_hwfn *p_hwfn, 568 struct qed_ptt *p_ptt, 569 struct qed_rdma_start_in_params *params) 570 { 571 int rc; 572 573 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA setup\n"); 574 575 spin_lock_init(&p_hwfn->p_rdma_info->lock); 576 577 qed_rdma_init_devinfo(p_hwfn, params); 578 qed_rdma_init_port(p_hwfn); 579 qed_rdma_init_events(p_hwfn, params); 580 581 rc = qed_rdma_reserve_lkey(p_hwfn); 582 if (rc) 583 return rc; 584 585 rc = qed_rdma_init_hw(p_hwfn, p_ptt); 586 if (rc) 587 return rc; 588 589 return qed_rdma_start_fw(p_hwfn, params, p_ptt); 590 } 591 592 static int qed_rdma_stop(void *rdma_cxt) 593 { 594 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 595 struct rdma_close_func_ramrod_data *p_ramrod; 596 struct qed_sp_init_data init_data; 597 struct qed_spq_entry *p_ent; 598 struct qed_ptt *p_ptt; 599 u32 ll2_ethertype_en; 600 int rc = -EBUSY; 601 602 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop\n"); 603 604 p_ptt = qed_ptt_acquire(p_hwfn); 605 if (!p_ptt) { 606 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Failed to acquire PTT\n"); 607 return rc; 608 } 609 610 /* Disable RoCE search */ 611 qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 0); 612 p_hwfn->b_rdma_enabled_in_prs = false; 613 614 qed_wr(p_hwfn, p_ptt, PRS_REG_ROCE_DEST_QP_MAX_PF, 0); 615 616 ll2_ethertype_en = qed_rd(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN); 617 618 qed_wr(p_hwfn, p_ptt, PRS_REG_LIGHT_L2_ETHERTYPE_EN, 619 (ll2_ethertype_en & 0xFFFE)); 620 621 qed_ptt_release(p_hwfn, p_ptt); 622 623 /* Get SPQ entry */ 624 memset(&init_data, 0, sizeof(init_data)); 625 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 626 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 627 628 /* Stop RoCE */ 629 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_FUNC_CLOSE, 630 p_hwfn->p_rdma_info->proto, &init_data); 631 if (rc) 632 goto out; 633 634 p_ramrod = &p_ent->ramrod.rdma_close_func; 635 636 p_ramrod->num_cnqs = p_hwfn->p_rdma_info->num_cnqs; 637 p_ramrod->cnq_start_offset = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM); 638 639 rc = qed_spq_post(p_hwfn, p_ent, NULL); 640 641 out: 642 qed_rdma_free(p_hwfn); 643 644 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA stop done, rc = %d\n", rc); 645 return rc; 646 } 647 648 static int qed_rdma_add_user(void *rdma_cxt, 649 struct qed_rdma_add_user_out_params *out_params) 650 { 651 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 652 u32 dpi_start_offset; 653 u32 returned_id = 0; 654 int rc; 655 656 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding User\n"); 657 658 /* Allocate DPI */ 659 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 660 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, 661 &returned_id); 662 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 663 664 out_params->dpi = (u16)returned_id; 665 666 /* Calculate the corresponding DPI address */ 667 dpi_start_offset = p_hwfn->dpi_start_offset; 668 669 out_params->dpi_addr = (u64)((u8 __iomem *)p_hwfn->doorbells + 670 dpi_start_offset + 671 ((out_params->dpi) * p_hwfn->dpi_size)); 672 673 out_params->dpi_phys_addr = p_hwfn->cdev->db_phys_addr + 674 dpi_start_offset + 675 ((out_params->dpi) * p_hwfn->dpi_size); 676 677 out_params->dpi_size = p_hwfn->dpi_size; 678 679 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Adding user - done, rc = %d\n", rc); 680 return rc; 681 } 682 683 static struct qed_rdma_port *qed_rdma_query_port(void *rdma_cxt) 684 { 685 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 686 struct qed_rdma_port *p_port = p_hwfn->p_rdma_info->port; 687 688 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA Query port\n"); 689 690 /* Link may have changed */ 691 p_port->port_state = p_hwfn->mcp_info->link_output.link_up ? 692 QED_RDMA_PORT_UP : QED_RDMA_PORT_DOWN; 693 694 p_port->link_speed = p_hwfn->mcp_info->link_output.speed; 695 696 return p_port; 697 } 698 699 static struct qed_rdma_device *qed_rdma_query_device(void *rdma_cxt) 700 { 701 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 702 703 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query device\n"); 704 705 /* Return struct with device parameters */ 706 return p_hwfn->p_rdma_info->dev; 707 } 708 709 static void qed_rdma_free_tid(void *rdma_cxt, u32 itid) 710 { 711 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 712 713 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 714 715 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 716 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tid_map, itid); 717 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 718 } 719 720 static void qed_rdma_cnq_prod_update(void *rdma_cxt, u8 qz_offset, u16 prod) 721 { 722 struct qed_hwfn *p_hwfn; 723 u16 qz_num; 724 u32 addr; 725 726 p_hwfn = (struct qed_hwfn *)rdma_cxt; 727 qz_num = p_hwfn->p_rdma_info->queue_zone_base + qz_offset; 728 addr = GTT_BAR0_MAP_REG_USDM_RAM + 729 USTORM_COMMON_QUEUE_CONS_OFFSET(qz_num); 730 731 REG_WR16(p_hwfn, addr, prod); 732 733 /* keep prod updates ordered */ 734 wmb(); 735 } 736 737 static int qed_fill_rdma_dev_info(struct qed_dev *cdev, 738 struct qed_dev_rdma_info *info) 739 { 740 memset(info, 0, sizeof(*info)); 741 742 info->rdma_type = QED_RDMA_TYPE_ROCE; 743 744 qed_fill_dev_info(cdev, &info->common); 745 746 return 0; 747 } 748 749 static int qed_rdma_get_sb_start(struct qed_dev *cdev) 750 { 751 int feat_num; 752 753 if (cdev->num_hwfns > 1) 754 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE); 755 else 756 feat_num = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_PF_L2_QUE) * 757 cdev->num_hwfns; 758 759 return feat_num; 760 } 761 762 static int qed_rdma_get_min_cnq_msix(struct qed_dev *cdev) 763 { 764 int n_cnq = FEAT_NUM(QED_LEADING_HWFN(cdev), QED_RDMA_CNQ); 765 int n_msix = cdev->int_params.rdma_msix_cnt; 766 767 return min_t(int, n_cnq, n_msix); 768 } 769 770 static int qed_rdma_set_int(struct qed_dev *cdev, u16 cnt) 771 { 772 int limit = 0; 773 774 /* Mark the fastpath as free/used */ 775 cdev->int_params.fp_initialized = cnt ? true : false; 776 777 if (cdev->int_params.out.int_mode != QED_INT_MODE_MSIX) { 778 DP_ERR(cdev, 779 "qed roce supports only MSI-X interrupts (detected %d).\n", 780 cdev->int_params.out.int_mode); 781 return -EINVAL; 782 } else if (cdev->int_params.fp_msix_cnt) { 783 limit = cdev->int_params.rdma_msix_cnt; 784 } 785 786 if (!limit) 787 return -ENOMEM; 788 789 return min_t(int, cnt, limit); 790 } 791 792 static int qed_rdma_get_int(struct qed_dev *cdev, struct qed_int_info *info) 793 { 794 memset(info, 0, sizeof(*info)); 795 796 if (!cdev->int_params.fp_initialized) { 797 DP_INFO(cdev, 798 "Protocol driver requested interrupt information, but its support is not yet configured\n"); 799 return -EINVAL; 800 } 801 802 if (cdev->int_params.out.int_mode == QED_INT_MODE_MSIX) { 803 int msix_base = cdev->int_params.rdma_msix_base; 804 805 info->msix_cnt = cdev->int_params.rdma_msix_cnt; 806 info->msix = &cdev->int_params.msix_table[msix_base]; 807 808 DP_VERBOSE(cdev, QED_MSG_RDMA, "msix_cnt = %d msix_base=%d\n", 809 info->msix_cnt, msix_base); 810 } 811 812 return 0; 813 } 814 815 static int qed_rdma_alloc_pd(void *rdma_cxt, u16 *pd) 816 { 817 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 818 u32 returned_id; 819 int rc; 820 821 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD\n"); 822 823 /* Allocates an unused protection domain */ 824 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 825 rc = qed_rdma_bmap_alloc_id(p_hwfn, 826 &p_hwfn->p_rdma_info->pd_map, &returned_id); 827 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 828 829 *pd = (u16)returned_id; 830 831 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Alloc PD - done, rc = %d\n", rc); 832 return rc; 833 } 834 835 static void qed_rdma_free_pd(void *rdma_cxt, u16 pd) 836 { 837 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 838 839 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "pd = %08x\n", pd); 840 841 /* Returns a previously allocated protection domain for reuse */ 842 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 843 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->pd_map, pd); 844 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 845 } 846 847 static enum qed_rdma_toggle_bit 848 qed_rdma_toggle_bit_create_resize_cq(struct qed_hwfn *p_hwfn, u16 icid) 849 { 850 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 851 enum qed_rdma_toggle_bit toggle_bit; 852 u32 bmap_id; 853 854 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", icid); 855 856 /* the function toggle the bit that is related to a given icid 857 * and returns the new toggle bit's value 858 */ 859 bmap_id = icid - qed_cxt_get_proto_cid_start(p_hwfn, p_info->proto); 860 861 spin_lock_bh(&p_info->lock); 862 toggle_bit = !test_and_change_bit(bmap_id, 863 p_info->toggle_bits.bitmap); 864 spin_unlock_bh(&p_info->lock); 865 866 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QED_RDMA_TOGGLE_BIT_= %d\n", 867 toggle_bit); 868 869 return toggle_bit; 870 } 871 872 static int qed_rdma_create_cq(void *rdma_cxt, 873 struct qed_rdma_create_cq_in_params *params, 874 u16 *icid) 875 { 876 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 877 struct qed_rdma_info *p_info = p_hwfn->p_rdma_info; 878 struct rdma_create_cq_ramrod_data *p_ramrod; 879 enum qed_rdma_toggle_bit toggle_bit; 880 struct qed_sp_init_data init_data; 881 struct qed_spq_entry *p_ent; 882 u32 returned_id, start_cid; 883 int rc; 884 885 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "cq_handle = %08x%08x\n", 886 params->cq_handle_hi, params->cq_handle_lo); 887 888 /* Allocate icid */ 889 spin_lock_bh(&p_info->lock); 890 rc = qed_rdma_bmap_alloc_id(p_hwfn, 891 &p_info->cq_map, &returned_id); 892 spin_unlock_bh(&p_info->lock); 893 894 if (rc) { 895 DP_NOTICE(p_hwfn, "Can't create CQ, rc = %d\n", rc); 896 return rc; 897 } 898 899 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, 900 p_info->proto); 901 *icid = returned_id + start_cid; 902 903 /* Check if icid requires a page allocation */ 904 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *icid); 905 if (rc) 906 goto err; 907 908 /* Get SPQ entry */ 909 memset(&init_data, 0, sizeof(init_data)); 910 init_data.cid = *icid; 911 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 912 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 913 914 /* Send create CQ ramrod */ 915 rc = qed_sp_init_request(p_hwfn, &p_ent, 916 RDMA_RAMROD_CREATE_CQ, 917 p_info->proto, &init_data); 918 if (rc) 919 goto err; 920 921 p_ramrod = &p_ent->ramrod.rdma_create_cq; 922 923 p_ramrod->cq_handle.hi = cpu_to_le32(params->cq_handle_hi); 924 p_ramrod->cq_handle.lo = cpu_to_le32(params->cq_handle_lo); 925 p_ramrod->dpi = cpu_to_le16(params->dpi); 926 p_ramrod->is_two_level_pbl = params->pbl_two_level; 927 p_ramrod->max_cqes = cpu_to_le32(params->cq_size); 928 DMA_REGPAIR_LE(p_ramrod->pbl_addr, params->pbl_ptr); 929 p_ramrod->pbl_num_pages = cpu_to_le16(params->pbl_num_pages); 930 p_ramrod->cnq_id = (u8)RESC_START(p_hwfn, QED_RDMA_CNQ_RAM) + 931 params->cnq_id; 932 p_ramrod->int_timeout = params->int_timeout; 933 934 /* toggle the bit for every resize or create cq for a given icid */ 935 toggle_bit = qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 936 937 p_ramrod->toggle_bit = toggle_bit; 938 939 rc = qed_spq_post(p_hwfn, p_ent, NULL); 940 if (rc) { 941 /* restore toggle bit */ 942 qed_rdma_toggle_bit_create_resize_cq(p_hwfn, *icid); 943 goto err; 944 } 945 946 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Created CQ, rc = %d\n", rc); 947 return rc; 948 949 err: 950 /* release allocated icid */ 951 qed_bmap_release_id(p_hwfn, &p_info->cq_map, returned_id); 952 DP_NOTICE(p_hwfn, "Create CQ failed, rc = %d\n", rc); 953 954 return rc; 955 } 956 957 static int 958 qed_rdma_destroy_cq(void *rdma_cxt, 959 struct qed_rdma_destroy_cq_in_params *in_params, 960 struct qed_rdma_destroy_cq_out_params *out_params) 961 { 962 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 963 struct rdma_destroy_cq_output_params *p_ramrod_res; 964 struct rdma_destroy_cq_ramrod_data *p_ramrod; 965 struct qed_sp_init_data init_data; 966 struct qed_spq_entry *p_ent; 967 dma_addr_t ramrod_res_phys; 968 int rc = -ENOMEM; 969 970 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", in_params->icid); 971 972 p_ramrod_res = 973 (struct rdma_destroy_cq_output_params *) 974 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 975 sizeof(struct rdma_destroy_cq_output_params), 976 &ramrod_res_phys, GFP_KERNEL); 977 if (!p_ramrod_res) { 978 DP_NOTICE(p_hwfn, 979 "qed destroy cq failed: cannot allocate memory (ramrod)\n"); 980 return rc; 981 } 982 983 /* Get SPQ entry */ 984 memset(&init_data, 0, sizeof(init_data)); 985 init_data.cid = in_params->icid; 986 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 987 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 988 989 /* Send destroy CQ ramrod */ 990 rc = qed_sp_init_request(p_hwfn, &p_ent, 991 RDMA_RAMROD_DESTROY_CQ, 992 p_hwfn->p_rdma_info->proto, &init_data); 993 if (rc) 994 goto err; 995 996 p_ramrod = &p_ent->ramrod.rdma_destroy_cq; 997 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 998 999 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1000 if (rc) 1001 goto err; 1002 1003 out_params->num_cq_notif = le16_to_cpu(p_ramrod_res->cnq_num); 1004 1005 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1006 sizeof(struct rdma_destroy_cq_output_params), 1007 p_ramrod_res, ramrod_res_phys); 1008 1009 /* Free icid */ 1010 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1011 1012 qed_bmap_release_id(p_hwfn, 1013 &p_hwfn->p_rdma_info->cq_map, 1014 (in_params->icid - 1015 qed_cxt_get_proto_cid_start(p_hwfn, 1016 p_hwfn-> 1017 p_rdma_info->proto))); 1018 1019 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1020 1021 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroyed CQ, rc = %d\n", rc); 1022 return rc; 1023 1024 err: dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1025 sizeof(struct rdma_destroy_cq_output_params), 1026 p_ramrod_res, ramrod_res_phys); 1027 1028 return rc; 1029 } 1030 1031 static void qed_rdma_set_fw_mac(u16 *p_fw_mac, u8 *p_qed_mac) 1032 { 1033 p_fw_mac[0] = cpu_to_le16((p_qed_mac[0] << 8) + p_qed_mac[1]); 1034 p_fw_mac[1] = cpu_to_le16((p_qed_mac[2] << 8) + p_qed_mac[3]); 1035 p_fw_mac[2] = cpu_to_le16((p_qed_mac[4] << 8) + p_qed_mac[5]); 1036 } 1037 1038 static void qed_rdma_copy_gids(struct qed_rdma_qp *qp, __le32 *src_gid, 1039 __le32 *dst_gid) 1040 { 1041 u32 i; 1042 1043 if (qp->roce_mode == ROCE_V2_IPV4) { 1044 /* The IPv4 addresses shall be aligned to the highest word. 1045 * The lower words must be zero. 1046 */ 1047 memset(src_gid, 0, sizeof(union qed_gid)); 1048 memset(dst_gid, 0, sizeof(union qed_gid)); 1049 src_gid[3] = cpu_to_le32(qp->sgid.ipv4_addr); 1050 dst_gid[3] = cpu_to_le32(qp->dgid.ipv4_addr); 1051 } else { 1052 /* GIDs and IPv6 addresses coincide in location and size */ 1053 for (i = 0; i < ARRAY_SIZE(qp->sgid.dwords); i++) { 1054 src_gid[i] = cpu_to_le32(qp->sgid.dwords[i]); 1055 dst_gid[i] = cpu_to_le32(qp->dgid.dwords[i]); 1056 } 1057 } 1058 } 1059 1060 static enum roce_flavor qed_roce_mode_to_flavor(enum roce_mode roce_mode) 1061 { 1062 enum roce_flavor flavor; 1063 1064 switch (roce_mode) { 1065 case ROCE_V1: 1066 flavor = PLAIN_ROCE; 1067 break; 1068 case ROCE_V2_IPV4: 1069 flavor = RROCE_IPV4; 1070 break; 1071 case ROCE_V2_IPV6: 1072 flavor = ROCE_V2_IPV6; 1073 break; 1074 default: 1075 flavor = MAX_ROCE_MODE; 1076 break; 1077 } 1078 return flavor; 1079 } 1080 1081 static int qed_roce_alloc_cid(struct qed_hwfn *p_hwfn, u16 *cid) 1082 { 1083 struct qed_rdma_info *p_rdma_info = p_hwfn->p_rdma_info; 1084 u32 responder_icid; 1085 u32 requester_icid; 1086 int rc; 1087 1088 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1089 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1090 &responder_icid); 1091 if (rc) { 1092 spin_unlock_bh(&p_rdma_info->lock); 1093 return rc; 1094 } 1095 1096 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_rdma_info->cid_map, 1097 &requester_icid); 1098 1099 spin_unlock_bh(&p_rdma_info->lock); 1100 if (rc) 1101 goto err; 1102 1103 /* the two icid's should be adjacent */ 1104 if ((requester_icid - responder_icid) != 1) { 1105 DP_NOTICE(p_hwfn, "Failed to allocate two adjacent qp's'\n"); 1106 rc = -EINVAL; 1107 goto err; 1108 } 1109 1110 responder_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1111 p_rdma_info->proto); 1112 requester_icid += qed_cxt_get_proto_cid_start(p_hwfn, 1113 p_rdma_info->proto); 1114 1115 /* If these icids require a new ILT line allocate DMA-able context for 1116 * an ILT page 1117 */ 1118 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, responder_icid); 1119 if (rc) 1120 goto err; 1121 1122 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, requester_icid); 1123 if (rc) 1124 goto err; 1125 1126 *cid = (u16)responder_icid; 1127 return rc; 1128 1129 err: 1130 spin_lock_bh(&p_rdma_info->lock); 1131 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, responder_icid); 1132 qed_bmap_release_id(p_hwfn, &p_rdma_info->cid_map, requester_icid); 1133 1134 spin_unlock_bh(&p_rdma_info->lock); 1135 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1136 "Allocate CID - failed, rc = %d\n", rc); 1137 return rc; 1138 } 1139 1140 static int qed_roce_sp_create_responder(struct qed_hwfn *p_hwfn, 1141 struct qed_rdma_qp *qp) 1142 { 1143 struct roce_create_qp_resp_ramrod_data *p_ramrod; 1144 struct qed_sp_init_data init_data; 1145 union qed_qm_pq_params qm_params; 1146 enum roce_flavor roce_flavor; 1147 struct qed_spq_entry *p_ent; 1148 u16 physical_queue0 = 0; 1149 int rc; 1150 1151 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1152 1153 /* Allocate DMA-able memory for IRQ */ 1154 qp->irq_num_pages = 1; 1155 qp->irq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1156 RDMA_RING_PAGE_SIZE, 1157 &qp->irq_phys_addr, GFP_KERNEL); 1158 if (!qp->irq) { 1159 rc = -ENOMEM; 1160 DP_NOTICE(p_hwfn, 1161 "qed create responder failed: cannot allocate memory (irq). rc = %d\n", 1162 rc); 1163 return rc; 1164 } 1165 1166 /* Get SPQ entry */ 1167 memset(&init_data, 0, sizeof(init_data)); 1168 init_data.cid = qp->icid; 1169 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1170 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1171 1172 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_CREATE_QP, 1173 PROTOCOLID_ROCE, &init_data); 1174 if (rc) 1175 goto err; 1176 1177 p_ramrod = &p_ent->ramrod.roce_create_qp_resp; 1178 1179 p_ramrod->flags = 0; 1180 1181 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1182 SET_FIELD(p_ramrod->flags, 1183 ROCE_CREATE_QP_RESP_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1184 1185 SET_FIELD(p_ramrod->flags, 1186 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1187 qp->incoming_rdma_read_en); 1188 1189 SET_FIELD(p_ramrod->flags, 1190 ROCE_CREATE_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1191 qp->incoming_rdma_write_en); 1192 1193 SET_FIELD(p_ramrod->flags, 1194 ROCE_CREATE_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1195 qp->incoming_atomic_en); 1196 1197 SET_FIELD(p_ramrod->flags, 1198 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1199 qp->e2e_flow_control_en); 1200 1201 SET_FIELD(p_ramrod->flags, 1202 ROCE_CREATE_QP_RESP_RAMROD_DATA_SRQ_FLG, qp->use_srq); 1203 1204 SET_FIELD(p_ramrod->flags, 1205 ROCE_CREATE_QP_RESP_RAMROD_DATA_RESERVED_KEY_EN, 1206 qp->fmr_and_reserved_lkey); 1207 1208 SET_FIELD(p_ramrod->flags, 1209 ROCE_CREATE_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1210 qp->min_rnr_nak_timer); 1211 1212 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1213 p_ramrod->traffic_class = qp->traffic_class_tos; 1214 p_ramrod->hop_limit = qp->hop_limit_ttl; 1215 p_ramrod->irq_num_pages = qp->irq_num_pages; 1216 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1217 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1218 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1219 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1220 p_ramrod->initial_psn = cpu_to_le32(qp->rq_psn); 1221 p_ramrod->pd = cpu_to_le16(qp->pd); 1222 p_ramrod->rq_num_pages = cpu_to_le16(qp->rq_num_pages); 1223 DMA_REGPAIR_LE(p_ramrod->rq_pbl_addr, qp->rq_pbl_ptr); 1224 DMA_REGPAIR_LE(p_ramrod->irq_pbl_addr, qp->irq_phys_addr); 1225 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1226 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1227 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1228 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1229 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1230 p_ramrod->stats_counter_id = p_hwfn->rel_pf_id; 1231 p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | 1232 qp->rq_cq_id); 1233 1234 memset(&qm_params, 0, sizeof(qm_params)); 1235 qm_params.roce.qpid = qp->icid >> 1; 1236 physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params); 1237 1238 p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0); 1239 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1240 1241 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1242 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1243 1244 p_ramrod->udp_src_port = qp->udp_src_port; 1245 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1246 p_ramrod->srq_id.srq_idx = cpu_to_le16(qp->srq_id); 1247 p_ramrod->srq_id.opaque_fid = cpu_to_le16(p_hwfn->hw_info.opaque_fid); 1248 1249 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1250 qp->stats_queue; 1251 1252 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1253 1254 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d physical_queue0 = 0x%x\n", 1255 rc, physical_queue0); 1256 1257 if (rc) 1258 goto err; 1259 1260 qp->resp_offloaded = true; 1261 1262 return rc; 1263 1264 err: 1265 DP_NOTICE(p_hwfn, "create responder - failed, rc = %d\n", rc); 1266 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1267 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1268 qp->irq, qp->irq_phys_addr); 1269 1270 return rc; 1271 } 1272 1273 static int qed_roce_sp_create_requester(struct qed_hwfn *p_hwfn, 1274 struct qed_rdma_qp *qp) 1275 { 1276 struct roce_create_qp_req_ramrod_data *p_ramrod; 1277 struct qed_sp_init_data init_data; 1278 union qed_qm_pq_params qm_params; 1279 enum roce_flavor roce_flavor; 1280 struct qed_spq_entry *p_ent; 1281 u16 physical_queue0 = 0; 1282 int rc; 1283 1284 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1285 1286 /* Allocate DMA-able memory for ORQ */ 1287 qp->orq_num_pages = 1; 1288 qp->orq = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1289 RDMA_RING_PAGE_SIZE, 1290 &qp->orq_phys_addr, GFP_KERNEL); 1291 if (!qp->orq) { 1292 rc = -ENOMEM; 1293 DP_NOTICE(p_hwfn, 1294 "qed create requester failed: cannot allocate memory (orq). rc = %d\n", 1295 rc); 1296 return rc; 1297 } 1298 1299 /* Get SPQ entry */ 1300 memset(&init_data, 0, sizeof(init_data)); 1301 init_data.cid = qp->icid + 1; 1302 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1303 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1304 1305 rc = qed_sp_init_request(p_hwfn, &p_ent, 1306 ROCE_RAMROD_CREATE_QP, 1307 PROTOCOLID_ROCE, &init_data); 1308 if (rc) 1309 goto err; 1310 1311 p_ramrod = &p_ent->ramrod.roce_create_qp_req; 1312 1313 p_ramrod->flags = 0; 1314 1315 roce_flavor = qed_roce_mode_to_flavor(qp->roce_mode); 1316 SET_FIELD(p_ramrod->flags, 1317 ROCE_CREATE_QP_REQ_RAMROD_DATA_ROCE_FLAVOR, roce_flavor); 1318 1319 SET_FIELD(p_ramrod->flags, 1320 ROCE_CREATE_QP_REQ_RAMROD_DATA_FMR_AND_RESERVED_EN, 1321 qp->fmr_and_reserved_lkey); 1322 1323 SET_FIELD(p_ramrod->flags, 1324 ROCE_CREATE_QP_REQ_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); 1325 1326 SET_FIELD(p_ramrod->flags, 1327 ROCE_CREATE_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1328 1329 SET_FIELD(p_ramrod->flags, 1330 ROCE_CREATE_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1331 qp->rnr_retry_cnt); 1332 1333 p_ramrod->max_ord = qp->max_rd_atomic_req; 1334 p_ramrod->traffic_class = qp->traffic_class_tos; 1335 p_ramrod->hop_limit = qp->hop_limit_ttl; 1336 p_ramrod->orq_num_pages = qp->orq_num_pages; 1337 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1338 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1339 p_ramrod->dst_qp_id = cpu_to_le32(qp->dest_qp); 1340 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1341 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1342 p_ramrod->initial_psn = cpu_to_le32(qp->sq_psn); 1343 p_ramrod->pd = cpu_to_le16(qp->pd); 1344 p_ramrod->sq_num_pages = cpu_to_le16(qp->sq_num_pages); 1345 DMA_REGPAIR_LE(p_ramrod->sq_pbl_addr, qp->sq_pbl_ptr); 1346 DMA_REGPAIR_LE(p_ramrod->orq_pbl_addr, qp->orq_phys_addr); 1347 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1348 p_ramrod->qp_handle_for_async.hi = cpu_to_le32(qp->qp_handle_async.hi); 1349 p_ramrod->qp_handle_for_async.lo = cpu_to_le32(qp->qp_handle_async.lo); 1350 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 1351 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 1352 p_ramrod->stats_counter_id = p_hwfn->rel_pf_id; 1353 p_ramrod->cq_cid = cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | 1354 qp->sq_cq_id); 1355 1356 memset(&qm_params, 0, sizeof(qm_params)); 1357 qm_params.roce.qpid = qp->icid >> 1; 1358 physical_queue0 = qed_get_qm_pq(p_hwfn, PROTOCOLID_ROCE, &qm_params); 1359 1360 p_ramrod->physical_queue0 = cpu_to_le16(physical_queue0); 1361 p_ramrod->dpi = cpu_to_le16(qp->dpi); 1362 1363 qed_rdma_set_fw_mac(p_ramrod->remote_mac_addr, qp->remote_mac_addr); 1364 qed_rdma_set_fw_mac(p_ramrod->local_mac_addr, qp->local_mac_addr); 1365 1366 p_ramrod->udp_src_port = qp->udp_src_port; 1367 p_ramrod->vlan_id = cpu_to_le16(qp->vlan_id); 1368 p_ramrod->stats_counter_id = RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + 1369 qp->stats_queue; 1370 1371 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1372 1373 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 1374 1375 if (rc) 1376 goto err; 1377 1378 qp->req_offloaded = true; 1379 1380 return rc; 1381 1382 err: 1383 DP_NOTICE(p_hwfn, "Create requested - failed, rc = %d\n", rc); 1384 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1385 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1386 qp->orq, qp->orq_phys_addr); 1387 return rc; 1388 } 1389 1390 static int qed_roce_sp_modify_responder(struct qed_hwfn *p_hwfn, 1391 struct qed_rdma_qp *qp, 1392 bool move_to_err, u32 modify_flags) 1393 { 1394 struct roce_modify_qp_resp_ramrod_data *p_ramrod; 1395 struct qed_sp_init_data init_data; 1396 struct qed_spq_entry *p_ent; 1397 int rc; 1398 1399 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1400 1401 if (move_to_err && !qp->resp_offloaded) 1402 return 0; 1403 1404 /* Get SPQ entry */ 1405 memset(&init_data, 0, sizeof(init_data)); 1406 init_data.cid = qp->icid; 1407 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1408 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1409 1410 rc = qed_sp_init_request(p_hwfn, &p_ent, 1411 ROCE_EVENT_MODIFY_QP, 1412 PROTOCOLID_ROCE, &init_data); 1413 if (rc) { 1414 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1415 return rc; 1416 } 1417 1418 p_ramrod = &p_ent->ramrod.roce_modify_qp_resp; 1419 1420 p_ramrod->flags = 0; 1421 1422 SET_FIELD(p_ramrod->flags, 1423 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1424 1425 SET_FIELD(p_ramrod->flags, 1426 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_RD_EN, 1427 qp->incoming_rdma_read_en); 1428 1429 SET_FIELD(p_ramrod->flags, 1430 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_WR_EN, 1431 qp->incoming_rdma_write_en); 1432 1433 SET_FIELD(p_ramrod->flags, 1434 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ATOMIC_EN, 1435 qp->incoming_atomic_en); 1436 1437 SET_FIELD(p_ramrod->flags, 1438 ROCE_CREATE_QP_RESP_RAMROD_DATA_E2E_FLOW_CONTROL_EN, 1439 qp->e2e_flow_control_en); 1440 1441 SET_FIELD(p_ramrod->flags, 1442 ROCE_MODIFY_QP_RESP_RAMROD_DATA_RDMA_OPS_EN_FLG, 1443 GET_FIELD(modify_flags, 1444 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)); 1445 1446 SET_FIELD(p_ramrod->flags, 1447 ROCE_MODIFY_QP_RESP_RAMROD_DATA_P_KEY_FLG, 1448 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1449 1450 SET_FIELD(p_ramrod->flags, 1451 ROCE_MODIFY_QP_RESP_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1452 GET_FIELD(modify_flags, 1453 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1454 1455 SET_FIELD(p_ramrod->flags, 1456 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MAX_IRD_FLG, 1457 GET_FIELD(modify_flags, 1458 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)); 1459 1460 SET_FIELD(p_ramrod->flags, 1461 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER_FLG, 1462 GET_FIELD(modify_flags, 1463 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)); 1464 1465 p_ramrod->fields = 0; 1466 SET_FIELD(p_ramrod->fields, 1467 ROCE_MODIFY_QP_RESP_RAMROD_DATA_MIN_RNR_NAK_TIMER, 1468 qp->min_rnr_nak_timer); 1469 1470 p_ramrod->max_ird = qp->max_rd_atomic_resp; 1471 p_ramrod->traffic_class = qp->traffic_class_tos; 1472 p_ramrod->hop_limit = qp->hop_limit_ttl; 1473 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1474 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1475 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1476 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1477 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1478 1479 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify responder, rc = %d\n", rc); 1480 return rc; 1481 } 1482 1483 static int qed_roce_sp_modify_requester(struct qed_hwfn *p_hwfn, 1484 struct qed_rdma_qp *qp, 1485 bool move_to_sqd, 1486 bool move_to_err, u32 modify_flags) 1487 { 1488 struct roce_modify_qp_req_ramrod_data *p_ramrod; 1489 struct qed_sp_init_data init_data; 1490 struct qed_spq_entry *p_ent; 1491 int rc; 1492 1493 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1494 1495 if (move_to_err && !(qp->req_offloaded)) 1496 return 0; 1497 1498 /* Get SPQ entry */ 1499 memset(&init_data, 0, sizeof(init_data)); 1500 init_data.cid = qp->icid + 1; 1501 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1502 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1503 1504 rc = qed_sp_init_request(p_hwfn, &p_ent, 1505 ROCE_EVENT_MODIFY_QP, 1506 PROTOCOLID_ROCE, &init_data); 1507 if (rc) { 1508 DP_NOTICE(p_hwfn, "rc = %d\n", rc); 1509 return rc; 1510 } 1511 1512 p_ramrod = &p_ent->ramrod.roce_modify_qp_req; 1513 1514 p_ramrod->flags = 0; 1515 1516 SET_FIELD(p_ramrod->flags, 1517 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_ERR_FLG, move_to_err); 1518 1519 SET_FIELD(p_ramrod->flags, 1520 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MOVE_TO_SQD_FLG, move_to_sqd); 1521 1522 SET_FIELD(p_ramrod->flags, 1523 ROCE_MODIFY_QP_REQ_RAMROD_DATA_EN_SQD_ASYNC_NOTIFY, 1524 qp->sqd_async); 1525 1526 SET_FIELD(p_ramrod->flags, 1527 ROCE_MODIFY_QP_REQ_RAMROD_DATA_P_KEY_FLG, 1528 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)); 1529 1530 SET_FIELD(p_ramrod->flags, 1531 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ADDRESS_VECTOR_FLG, 1532 GET_FIELD(modify_flags, 1533 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)); 1534 1535 SET_FIELD(p_ramrod->flags, 1536 ROCE_MODIFY_QP_REQ_RAMROD_DATA_MAX_ORD_FLG, 1537 GET_FIELD(modify_flags, 1538 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)); 1539 1540 SET_FIELD(p_ramrod->flags, 1541 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT_FLG, 1542 GET_FIELD(modify_flags, 1543 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)); 1544 1545 SET_FIELD(p_ramrod->flags, 1546 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT_FLG, 1547 GET_FIELD(modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)); 1548 1549 SET_FIELD(p_ramrod->flags, 1550 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ACK_TIMEOUT_FLG, 1551 GET_FIELD(modify_flags, 1552 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)); 1553 1554 p_ramrod->fields = 0; 1555 SET_FIELD(p_ramrod->fields, 1556 ROCE_MODIFY_QP_REQ_RAMROD_DATA_ERR_RETRY_CNT, qp->retry_cnt); 1557 1558 SET_FIELD(p_ramrod->fields, 1559 ROCE_MODIFY_QP_REQ_RAMROD_DATA_RNR_NAK_CNT, 1560 qp->rnr_retry_cnt); 1561 1562 p_ramrod->max_ord = qp->max_rd_atomic_req; 1563 p_ramrod->traffic_class = qp->traffic_class_tos; 1564 p_ramrod->hop_limit = qp->hop_limit_ttl; 1565 p_ramrod->p_key = cpu_to_le16(qp->pkey); 1566 p_ramrod->flow_label = cpu_to_le32(qp->flow_label); 1567 p_ramrod->ack_timeout_val = cpu_to_le32(qp->ack_timeout); 1568 p_ramrod->mtu = cpu_to_le16(qp->mtu); 1569 qed_rdma_copy_gids(qp, p_ramrod->src_gid, p_ramrod->dst_gid); 1570 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1571 1572 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify requester, rc = %d\n", rc); 1573 return rc; 1574 } 1575 1576 static int qed_roce_sp_destroy_qp_responder(struct qed_hwfn *p_hwfn, 1577 struct qed_rdma_qp *qp, 1578 u32 *num_invalidated_mw) 1579 { 1580 struct roce_destroy_qp_resp_output_params *p_ramrod_res; 1581 struct roce_destroy_qp_resp_ramrod_data *p_ramrod; 1582 struct qed_sp_init_data init_data; 1583 struct qed_spq_entry *p_ent; 1584 dma_addr_t ramrod_res_phys; 1585 int rc; 1586 1587 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1588 1589 if (!qp->resp_offloaded) 1590 return 0; 1591 1592 /* Get SPQ entry */ 1593 memset(&init_data, 0, sizeof(init_data)); 1594 init_data.cid = qp->icid; 1595 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1596 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1597 1598 rc = qed_sp_init_request(p_hwfn, &p_ent, 1599 ROCE_RAMROD_DESTROY_QP, 1600 PROTOCOLID_ROCE, &init_data); 1601 if (rc) 1602 return rc; 1603 1604 p_ramrod = &p_ent->ramrod.roce_destroy_qp_resp; 1605 1606 p_ramrod_res = (struct roce_destroy_qp_resp_output_params *) 1607 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1608 &ramrod_res_phys, GFP_KERNEL); 1609 1610 if (!p_ramrod_res) { 1611 rc = -ENOMEM; 1612 DP_NOTICE(p_hwfn, 1613 "qed destroy responder failed: cannot allocate memory (ramrod). rc = %d\n", 1614 rc); 1615 return rc; 1616 } 1617 1618 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1619 1620 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1621 if (rc) 1622 goto err; 1623 1624 *num_invalidated_mw = le32_to_cpu(p_ramrod_res->num_invalidated_mw); 1625 1626 /* Free IRQ - only if ramrod succeeded, in case FW is still using it */ 1627 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1628 qp->irq_num_pages * RDMA_RING_PAGE_SIZE, 1629 qp->irq, qp->irq_phys_addr); 1630 1631 qp->resp_offloaded = false; 1632 1633 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy responder, rc = %d\n", rc); 1634 1635 err: 1636 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1637 sizeof(struct roce_destroy_qp_resp_output_params), 1638 p_ramrod_res, ramrod_res_phys); 1639 1640 return rc; 1641 } 1642 1643 static int qed_roce_sp_destroy_qp_requester(struct qed_hwfn *p_hwfn, 1644 struct qed_rdma_qp *qp, 1645 u32 *num_bound_mw) 1646 { 1647 struct roce_destroy_qp_req_output_params *p_ramrod_res; 1648 struct roce_destroy_qp_req_ramrod_data *p_ramrod; 1649 struct qed_sp_init_data init_data; 1650 struct qed_spq_entry *p_ent; 1651 dma_addr_t ramrod_res_phys; 1652 int rc = -ENOMEM; 1653 1654 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1655 1656 if (!qp->req_offloaded) 1657 return 0; 1658 1659 p_ramrod_res = (struct roce_destroy_qp_req_output_params *) 1660 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1661 sizeof(*p_ramrod_res), 1662 &ramrod_res_phys, GFP_KERNEL); 1663 if (!p_ramrod_res) { 1664 DP_NOTICE(p_hwfn, 1665 "qed destroy requester failed: cannot allocate memory (ramrod)\n"); 1666 return rc; 1667 } 1668 1669 /* Get SPQ entry */ 1670 memset(&init_data, 0, sizeof(init_data)); 1671 init_data.cid = qp->icid + 1; 1672 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1673 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1674 1675 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_DESTROY_QP, 1676 PROTOCOLID_ROCE, &init_data); 1677 if (rc) 1678 goto err; 1679 1680 p_ramrod = &p_ent->ramrod.roce_destroy_qp_req; 1681 DMA_REGPAIR_LE(p_ramrod->output_params_addr, ramrod_res_phys); 1682 1683 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1684 if (rc) 1685 goto err; 1686 1687 *num_bound_mw = le32_to_cpu(p_ramrod_res->num_bound_mw); 1688 1689 /* Free ORQ - only if ramrod succeeded, in case FW is still using it */ 1690 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 1691 qp->orq_num_pages * RDMA_RING_PAGE_SIZE, 1692 qp->orq, qp->orq_phys_addr); 1693 1694 qp->req_offloaded = false; 1695 1696 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Destroy requester, rc = %d\n", rc); 1697 1698 err: 1699 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_ramrod_res), 1700 p_ramrod_res, ramrod_res_phys); 1701 1702 return rc; 1703 } 1704 1705 static int qed_roce_query_qp(struct qed_hwfn *p_hwfn, 1706 struct qed_rdma_qp *qp, 1707 struct qed_rdma_query_qp_out_params *out_params) 1708 { 1709 struct roce_query_qp_resp_output_params *p_resp_ramrod_res; 1710 struct roce_query_qp_req_output_params *p_req_ramrod_res; 1711 struct roce_query_qp_resp_ramrod_data *p_resp_ramrod; 1712 struct roce_query_qp_req_ramrod_data *p_req_ramrod; 1713 struct qed_sp_init_data init_data; 1714 dma_addr_t resp_ramrod_res_phys; 1715 dma_addr_t req_ramrod_res_phys; 1716 struct qed_spq_entry *p_ent; 1717 bool rq_err_state; 1718 bool sq_err_state; 1719 bool sq_draining; 1720 int rc = -ENOMEM; 1721 1722 if ((!(qp->resp_offloaded)) && (!(qp->req_offloaded))) { 1723 /* We can't send ramrod to the fw since this qp wasn't offloaded 1724 * to the fw yet 1725 */ 1726 out_params->draining = false; 1727 out_params->rq_psn = qp->rq_psn; 1728 out_params->sq_psn = qp->sq_psn; 1729 out_params->state = qp->cur_state; 1730 1731 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "No QPs as no offload\n"); 1732 return 0; 1733 } 1734 1735 if (!(qp->resp_offloaded)) { 1736 DP_NOTICE(p_hwfn, 1737 "The responder's qp should be offloded before requester's\n"); 1738 return -EINVAL; 1739 } 1740 1741 /* Send a query responder ramrod to FW to get RQ-PSN and state */ 1742 p_resp_ramrod_res = (struct roce_query_qp_resp_output_params *) 1743 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1744 sizeof(*p_resp_ramrod_res), 1745 &resp_ramrod_res_phys, GFP_KERNEL); 1746 if (!p_resp_ramrod_res) { 1747 DP_NOTICE(p_hwfn, 1748 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1749 return rc; 1750 } 1751 1752 /* Get SPQ entry */ 1753 memset(&init_data, 0, sizeof(init_data)); 1754 init_data.cid = qp->icid; 1755 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 1756 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 1757 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1758 PROTOCOLID_ROCE, &init_data); 1759 if (rc) 1760 goto err_resp; 1761 1762 p_resp_ramrod = &p_ent->ramrod.roce_query_qp_resp; 1763 DMA_REGPAIR_LE(p_resp_ramrod->output_params_addr, resp_ramrod_res_phys); 1764 1765 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1766 if (rc) 1767 goto err_resp; 1768 1769 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 1770 p_resp_ramrod_res, resp_ramrod_res_phys); 1771 1772 out_params->rq_psn = le32_to_cpu(p_resp_ramrod_res->psn); 1773 rq_err_state = GET_FIELD(le32_to_cpu(p_resp_ramrod_res->err_flag), 1774 ROCE_QUERY_QP_RESP_OUTPUT_PARAMS_ERROR_FLG); 1775 1776 if (!(qp->req_offloaded)) { 1777 /* Don't send query qp for the requester */ 1778 out_params->sq_psn = qp->sq_psn; 1779 out_params->draining = false; 1780 1781 if (rq_err_state) 1782 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1783 1784 out_params->state = qp->cur_state; 1785 1786 return 0; 1787 } 1788 1789 /* Send a query requester ramrod to FW to get SQ-PSN and state */ 1790 p_req_ramrod_res = (struct roce_query_qp_req_output_params *) 1791 dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1792 sizeof(*p_req_ramrod_res), 1793 &req_ramrod_res_phys, 1794 GFP_KERNEL); 1795 if (!p_req_ramrod_res) { 1796 rc = -ENOMEM; 1797 DP_NOTICE(p_hwfn, 1798 "qed query qp failed: cannot allocate memory (ramrod)\n"); 1799 return rc; 1800 } 1801 1802 /* Get SPQ entry */ 1803 init_data.cid = qp->icid + 1; 1804 rc = qed_sp_init_request(p_hwfn, &p_ent, ROCE_RAMROD_QUERY_QP, 1805 PROTOCOLID_ROCE, &init_data); 1806 if (rc) 1807 goto err_req; 1808 1809 p_req_ramrod = &p_ent->ramrod.roce_query_qp_req; 1810 DMA_REGPAIR_LE(p_req_ramrod->output_params_addr, req_ramrod_res_phys); 1811 1812 rc = qed_spq_post(p_hwfn, p_ent, NULL); 1813 if (rc) 1814 goto err_req; 1815 1816 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 1817 p_req_ramrod_res, req_ramrod_res_phys); 1818 1819 out_params->sq_psn = le32_to_cpu(p_req_ramrod_res->psn); 1820 sq_err_state = GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1821 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_ERR_FLG); 1822 sq_draining = 1823 GET_FIELD(le32_to_cpu(p_req_ramrod_res->flags), 1824 ROCE_QUERY_QP_REQ_OUTPUT_PARAMS_SQ_DRAINING_FLG); 1825 1826 out_params->draining = false; 1827 1828 if (rq_err_state) 1829 qp->cur_state = QED_ROCE_QP_STATE_ERR; 1830 else if (sq_err_state) 1831 qp->cur_state = QED_ROCE_QP_STATE_SQE; 1832 else if (sq_draining) 1833 out_params->draining = true; 1834 out_params->state = qp->cur_state; 1835 1836 return 0; 1837 1838 err_req: 1839 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_req_ramrod_res), 1840 p_req_ramrod_res, req_ramrod_res_phys); 1841 return rc; 1842 err_resp: 1843 dma_free_coherent(&p_hwfn->cdev->pdev->dev, sizeof(*p_resp_ramrod_res), 1844 p_resp_ramrod_res, resp_ramrod_res_phys); 1845 return rc; 1846 } 1847 1848 static int qed_roce_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 1849 { 1850 u32 num_invalidated_mw = 0; 1851 u32 num_bound_mw = 0; 1852 u32 start_cid; 1853 int rc; 1854 1855 /* Destroys the specified QP */ 1856 if ((qp->cur_state != QED_ROCE_QP_STATE_RESET) && 1857 (qp->cur_state != QED_ROCE_QP_STATE_ERR) && 1858 (qp->cur_state != QED_ROCE_QP_STATE_INIT)) { 1859 DP_NOTICE(p_hwfn, 1860 "QP must be in error, reset or init state before destroying it\n"); 1861 return -EINVAL; 1862 } 1863 1864 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, &num_invalidated_mw); 1865 if (rc) 1866 return rc; 1867 1868 /* Send destroy requester ramrod */ 1869 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, &num_bound_mw); 1870 if (rc) 1871 return rc; 1872 1873 if (num_invalidated_mw != num_bound_mw) { 1874 DP_NOTICE(p_hwfn, 1875 "number of invalidate memory windows is different from bounded ones\n"); 1876 return -EINVAL; 1877 } 1878 1879 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 1880 1881 start_cid = qed_cxt_get_proto_cid_start(p_hwfn, 1882 p_hwfn->p_rdma_info->proto); 1883 1884 /* Release responder's icid */ 1885 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1886 qp->icid - start_cid); 1887 1888 /* Release requester's icid */ 1889 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, 1890 qp->icid + 1 - start_cid); 1891 1892 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 1893 1894 return 0; 1895 } 1896 1897 static int qed_rdma_query_qp(void *rdma_cxt, 1898 struct qed_rdma_qp *qp, 1899 struct qed_rdma_query_qp_out_params *out_params) 1900 { 1901 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1902 int rc; 1903 1904 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1905 1906 /* The following fields are filled in from qp and not FW as they can't 1907 * be modified by FW 1908 */ 1909 out_params->mtu = qp->mtu; 1910 out_params->dest_qp = qp->dest_qp; 1911 out_params->incoming_atomic_en = qp->incoming_atomic_en; 1912 out_params->e2e_flow_control_en = qp->e2e_flow_control_en; 1913 out_params->incoming_rdma_read_en = qp->incoming_rdma_read_en; 1914 out_params->incoming_rdma_write_en = qp->incoming_rdma_write_en; 1915 out_params->dgid = qp->dgid; 1916 out_params->flow_label = qp->flow_label; 1917 out_params->hop_limit_ttl = qp->hop_limit_ttl; 1918 out_params->traffic_class_tos = qp->traffic_class_tos; 1919 out_params->timeout = qp->ack_timeout; 1920 out_params->rnr_retry = qp->rnr_retry_cnt; 1921 out_params->retry_cnt = qp->retry_cnt; 1922 out_params->min_rnr_nak_timer = qp->min_rnr_nak_timer; 1923 out_params->pkey_index = 0; 1924 out_params->max_rd_atomic = qp->max_rd_atomic_req; 1925 out_params->max_dest_rd_atomic = qp->max_rd_atomic_resp; 1926 out_params->sqd_async = qp->sqd_async; 1927 1928 rc = qed_roce_query_qp(p_hwfn, qp, out_params); 1929 1930 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Query QP, rc = %d\n", rc); 1931 return rc; 1932 } 1933 1934 static int qed_rdma_destroy_qp(void *rdma_cxt, struct qed_rdma_qp *qp) 1935 { 1936 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1937 int rc = 0; 1938 1939 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x\n", qp->icid); 1940 1941 rc = qed_roce_destroy_qp(p_hwfn, qp); 1942 1943 /* free qp params struct */ 1944 kfree(qp); 1945 1946 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP destroyed\n"); 1947 return rc; 1948 } 1949 1950 static struct qed_rdma_qp * 1951 qed_rdma_create_qp(void *rdma_cxt, 1952 struct qed_rdma_create_qp_in_params *in_params, 1953 struct qed_rdma_create_qp_out_params *out_params) 1954 { 1955 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 1956 struct qed_rdma_qp *qp; 1957 u8 max_stats_queues; 1958 int rc; 1959 1960 if (!rdma_cxt || !in_params || !out_params || !p_hwfn->p_rdma_info) { 1961 DP_ERR(p_hwfn->cdev, 1962 "qed roce create qp failed due to NULL entry (rdma_cxt=%p, in=%p, out=%p, roce_info=?\n", 1963 rdma_cxt, in_params, out_params); 1964 return NULL; 1965 } 1966 1967 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1968 "qed rdma create qp called with qp_handle = %08x%08x\n", 1969 in_params->qp_handle_hi, in_params->qp_handle_lo); 1970 1971 /* Some sanity checks... */ 1972 max_stats_queues = p_hwfn->p_rdma_info->dev->max_stats_queues; 1973 if (in_params->stats_queue >= max_stats_queues) { 1974 DP_ERR(p_hwfn->cdev, 1975 "qed rdma create qp failed due to invalid statistics queue %d. maximum is %d\n", 1976 in_params->stats_queue, max_stats_queues); 1977 return NULL; 1978 } 1979 1980 qp = kzalloc(sizeof(*qp), GFP_KERNEL); 1981 if (!qp) { 1982 DP_NOTICE(p_hwfn, "Failed to allocate qed_rdma_qp\n"); 1983 return NULL; 1984 } 1985 1986 rc = qed_roce_alloc_cid(p_hwfn, &qp->icid); 1987 qp->qpid = ((0xFF << 16) | qp->icid); 1988 1989 DP_INFO(p_hwfn, "ROCE qpid=%x\n", qp->qpid); 1990 1991 if (rc) { 1992 kfree(qp); 1993 return NULL; 1994 } 1995 1996 qp->cur_state = QED_ROCE_QP_STATE_RESET; 1997 qp->qp_handle.hi = cpu_to_le32(in_params->qp_handle_hi); 1998 qp->qp_handle.lo = cpu_to_le32(in_params->qp_handle_lo); 1999 qp->qp_handle_async.hi = cpu_to_le32(in_params->qp_handle_async_hi); 2000 qp->qp_handle_async.lo = cpu_to_le32(in_params->qp_handle_async_lo); 2001 qp->use_srq = in_params->use_srq; 2002 qp->signal_all = in_params->signal_all; 2003 qp->fmr_and_reserved_lkey = in_params->fmr_and_reserved_lkey; 2004 qp->pd = in_params->pd; 2005 qp->dpi = in_params->dpi; 2006 qp->sq_cq_id = in_params->sq_cq_id; 2007 qp->sq_num_pages = in_params->sq_num_pages; 2008 qp->sq_pbl_ptr = in_params->sq_pbl_ptr; 2009 qp->rq_cq_id = in_params->rq_cq_id; 2010 qp->rq_num_pages = in_params->rq_num_pages; 2011 qp->rq_pbl_ptr = in_params->rq_pbl_ptr; 2012 qp->srq_id = in_params->srq_id; 2013 qp->req_offloaded = false; 2014 qp->resp_offloaded = false; 2015 qp->e2e_flow_control_en = qp->use_srq ? false : true; 2016 qp->stats_queue = in_params->stats_queue; 2017 2018 out_params->icid = qp->icid; 2019 out_params->qp_id = qp->qpid; 2020 2021 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Create QP, rc = %d\n", rc); 2022 return qp; 2023 } 2024 2025 static int qed_roce_modify_qp(struct qed_hwfn *p_hwfn, 2026 struct qed_rdma_qp *qp, 2027 enum qed_roce_qp_state prev_state, 2028 struct qed_rdma_modify_qp_in_params *params) 2029 { 2030 u32 num_invalidated_mw = 0, num_bound_mw = 0; 2031 int rc = 0; 2032 2033 /* Perform additional operations according to the current state and the 2034 * next state 2035 */ 2036 if (((prev_state == QED_ROCE_QP_STATE_INIT) || 2037 (prev_state == QED_ROCE_QP_STATE_RESET)) && 2038 (qp->cur_state == QED_ROCE_QP_STATE_RTR)) { 2039 /* Init->RTR or Reset->RTR */ 2040 rc = qed_roce_sp_create_responder(p_hwfn, qp); 2041 return rc; 2042 } else if ((prev_state == QED_ROCE_QP_STATE_RTR) && 2043 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2044 /* RTR-> RTS */ 2045 rc = qed_roce_sp_create_requester(p_hwfn, qp); 2046 if (rc) 2047 return rc; 2048 2049 /* Send modify responder ramrod */ 2050 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2051 params->modify_flags); 2052 return rc; 2053 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2054 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2055 /* RTS->RTS */ 2056 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2057 params->modify_flags); 2058 if (rc) 2059 return rc; 2060 2061 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2062 params->modify_flags); 2063 return rc; 2064 } else if ((prev_state == QED_ROCE_QP_STATE_RTS) && 2065 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2066 /* RTS->SQD */ 2067 rc = qed_roce_sp_modify_requester(p_hwfn, qp, true, false, 2068 params->modify_flags); 2069 return rc; 2070 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2071 (qp->cur_state == QED_ROCE_QP_STATE_SQD)) { 2072 /* SQD->SQD */ 2073 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2074 params->modify_flags); 2075 if (rc) 2076 return rc; 2077 2078 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2079 params->modify_flags); 2080 return rc; 2081 } else if ((prev_state == QED_ROCE_QP_STATE_SQD) && 2082 (qp->cur_state == QED_ROCE_QP_STATE_RTS)) { 2083 /* SQD->RTS */ 2084 rc = qed_roce_sp_modify_responder(p_hwfn, qp, false, 2085 params->modify_flags); 2086 if (rc) 2087 return rc; 2088 2089 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, false, 2090 params->modify_flags); 2091 2092 return rc; 2093 } else if (qp->cur_state == QED_ROCE_QP_STATE_ERR || 2094 qp->cur_state == QED_ROCE_QP_STATE_SQE) { 2095 /* ->ERR */ 2096 rc = qed_roce_sp_modify_responder(p_hwfn, qp, true, 2097 params->modify_flags); 2098 if (rc) 2099 return rc; 2100 2101 rc = qed_roce_sp_modify_requester(p_hwfn, qp, false, true, 2102 params->modify_flags); 2103 return rc; 2104 } else if (qp->cur_state == QED_ROCE_QP_STATE_RESET) { 2105 /* Any state -> RESET */ 2106 2107 rc = qed_roce_sp_destroy_qp_responder(p_hwfn, qp, 2108 &num_invalidated_mw); 2109 if (rc) 2110 return rc; 2111 2112 rc = qed_roce_sp_destroy_qp_requester(p_hwfn, qp, 2113 &num_bound_mw); 2114 2115 if (num_invalidated_mw != num_bound_mw) { 2116 DP_NOTICE(p_hwfn, 2117 "number of invalidate memory windows is different from bounded ones\n"); 2118 return -EINVAL; 2119 } 2120 } else { 2121 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "0\n"); 2122 } 2123 2124 return rc; 2125 } 2126 2127 static int qed_rdma_modify_qp(void *rdma_cxt, 2128 struct qed_rdma_qp *qp, 2129 struct qed_rdma_modify_qp_in_params *params) 2130 { 2131 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2132 enum qed_roce_qp_state prev_state; 2133 int rc = 0; 2134 2135 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "icid = %08x params->new_state=%d\n", 2136 qp->icid, params->new_state); 2137 2138 if (rc) { 2139 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2140 return rc; 2141 } 2142 2143 if (GET_FIELD(params->modify_flags, 2144 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN)) { 2145 qp->incoming_rdma_read_en = params->incoming_rdma_read_en; 2146 qp->incoming_rdma_write_en = params->incoming_rdma_write_en; 2147 qp->incoming_atomic_en = params->incoming_atomic_en; 2148 } 2149 2150 /* Update QP structure with the updated values */ 2151 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_ROCE_MODE)) 2152 qp->roce_mode = params->roce_mode; 2153 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_PKEY)) 2154 qp->pkey = params->pkey; 2155 if (GET_FIELD(params->modify_flags, 2156 QED_ROCE_MODIFY_QP_VALID_E2E_FLOW_CONTROL_EN)) 2157 qp->e2e_flow_control_en = params->e2e_flow_control_en; 2158 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_DEST_QP)) 2159 qp->dest_qp = params->dest_qp; 2160 if (GET_FIELD(params->modify_flags, 2161 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR)) { 2162 /* Indicates that the following parameters have changed: 2163 * Traffic class, flow label, hop limit, source GID, 2164 * destination GID, loopback indicator 2165 */ 2166 qp->traffic_class_tos = params->traffic_class_tos; 2167 qp->flow_label = params->flow_label; 2168 qp->hop_limit_ttl = params->hop_limit_ttl; 2169 2170 qp->sgid = params->sgid; 2171 qp->dgid = params->dgid; 2172 qp->udp_src_port = 0; 2173 qp->vlan_id = params->vlan_id; 2174 qp->mtu = params->mtu; 2175 qp->lb_indication = params->lb_indication; 2176 memcpy((u8 *)&qp->remote_mac_addr[0], 2177 (u8 *)¶ms->remote_mac_addr[0], ETH_ALEN); 2178 if (params->use_local_mac) { 2179 memcpy((u8 *)&qp->local_mac_addr[0], 2180 (u8 *)¶ms->local_mac_addr[0], ETH_ALEN); 2181 } else { 2182 memcpy((u8 *)&qp->local_mac_addr[0], 2183 (u8 *)&p_hwfn->hw_info.hw_mac_addr, ETH_ALEN); 2184 } 2185 } 2186 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RQ_PSN)) 2187 qp->rq_psn = params->rq_psn; 2188 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_SQ_PSN)) 2189 qp->sq_psn = params->sq_psn; 2190 if (GET_FIELD(params->modify_flags, 2191 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ)) 2192 qp->max_rd_atomic_req = params->max_rd_atomic_req; 2193 if (GET_FIELD(params->modify_flags, 2194 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP)) 2195 qp->max_rd_atomic_resp = params->max_rd_atomic_resp; 2196 if (GET_FIELD(params->modify_flags, 2197 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT)) 2198 qp->ack_timeout = params->ack_timeout; 2199 if (GET_FIELD(params->modify_flags, QED_ROCE_MODIFY_QP_VALID_RETRY_CNT)) 2200 qp->retry_cnt = params->retry_cnt; 2201 if (GET_FIELD(params->modify_flags, 2202 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT)) 2203 qp->rnr_retry_cnt = params->rnr_retry_cnt; 2204 if (GET_FIELD(params->modify_flags, 2205 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER)) 2206 qp->min_rnr_nak_timer = params->min_rnr_nak_timer; 2207 2208 qp->sqd_async = params->sqd_async; 2209 2210 prev_state = qp->cur_state; 2211 if (GET_FIELD(params->modify_flags, 2212 QED_RDMA_MODIFY_QP_VALID_NEW_STATE)) { 2213 qp->cur_state = params->new_state; 2214 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "qp->cur_state=%d\n", 2215 qp->cur_state); 2216 } 2217 2218 rc = qed_roce_modify_qp(p_hwfn, qp, prev_state, params); 2219 2220 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Modify QP, rc = %d\n", rc); 2221 return rc; 2222 } 2223 2224 static int 2225 qed_rdma_register_tid(void *rdma_cxt, 2226 struct qed_rdma_register_tid_in_params *params) 2227 { 2228 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2229 struct rdma_register_tid_ramrod_data *p_ramrod; 2230 struct qed_sp_init_data init_data; 2231 struct qed_spq_entry *p_ent; 2232 enum rdma_tid_type tid_type; 2233 u8 fw_return_code; 2234 int rc; 2235 2236 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", params->itid); 2237 2238 /* Get SPQ entry */ 2239 memset(&init_data, 0, sizeof(init_data)); 2240 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2241 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2242 2243 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_REGISTER_MR, 2244 p_hwfn->p_rdma_info->proto, &init_data); 2245 if (rc) { 2246 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2247 return rc; 2248 } 2249 2250 if (p_hwfn->p_rdma_info->last_tid < params->itid) 2251 p_hwfn->p_rdma_info->last_tid = params->itid; 2252 2253 p_ramrod = &p_ent->ramrod.rdma_register_tid; 2254 2255 p_ramrod->flags = 0; 2256 SET_FIELD(p_ramrod->flags, 2257 RDMA_REGISTER_TID_RAMROD_DATA_TWO_LEVEL_PBL, 2258 params->pbl_two_level); 2259 2260 SET_FIELD(p_ramrod->flags, 2261 RDMA_REGISTER_TID_RAMROD_DATA_ZERO_BASED, params->zbva); 2262 2263 SET_FIELD(p_ramrod->flags, 2264 RDMA_REGISTER_TID_RAMROD_DATA_PHY_MR, params->phy_mr); 2265 2266 /* Don't initialize D/C field, as it may override other bits. */ 2267 if (!(params->tid_type == QED_RDMA_TID_FMR) && !(params->dma_mr)) 2268 SET_FIELD(p_ramrod->flags, 2269 RDMA_REGISTER_TID_RAMROD_DATA_PAGE_SIZE_LOG, 2270 params->page_size_log - 12); 2271 2272 SET_FIELD(p_ramrod->flags, 2273 RDMA_REGISTER_TID_RAMROD_DATA_MAX_ID, 2274 p_hwfn->p_rdma_info->last_tid); 2275 2276 SET_FIELD(p_ramrod->flags, 2277 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_READ, 2278 params->remote_read); 2279 2280 SET_FIELD(p_ramrod->flags, 2281 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_WRITE, 2282 params->remote_write); 2283 2284 SET_FIELD(p_ramrod->flags, 2285 RDMA_REGISTER_TID_RAMROD_DATA_REMOTE_ATOMIC, 2286 params->remote_atomic); 2287 2288 SET_FIELD(p_ramrod->flags, 2289 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_WRITE, 2290 params->local_write); 2291 2292 SET_FIELD(p_ramrod->flags, 2293 RDMA_REGISTER_TID_RAMROD_DATA_LOCAL_READ, params->local_read); 2294 2295 SET_FIELD(p_ramrod->flags, 2296 RDMA_REGISTER_TID_RAMROD_DATA_ENABLE_MW_BIND, 2297 params->mw_bind); 2298 2299 SET_FIELD(p_ramrod->flags1, 2300 RDMA_REGISTER_TID_RAMROD_DATA_PBL_PAGE_SIZE_LOG, 2301 params->pbl_page_size_log - 12); 2302 2303 SET_FIELD(p_ramrod->flags2, 2304 RDMA_REGISTER_TID_RAMROD_DATA_DMA_MR, params->dma_mr); 2305 2306 switch (params->tid_type) { 2307 case QED_RDMA_TID_REGISTERED_MR: 2308 tid_type = RDMA_TID_REGISTERED_MR; 2309 break; 2310 case QED_RDMA_TID_FMR: 2311 tid_type = RDMA_TID_FMR; 2312 break; 2313 case QED_RDMA_TID_MW_TYPE1: 2314 tid_type = RDMA_TID_MW_TYPE1; 2315 break; 2316 case QED_RDMA_TID_MW_TYPE2A: 2317 tid_type = RDMA_TID_MW_TYPE2A; 2318 break; 2319 default: 2320 rc = -EINVAL; 2321 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2322 return rc; 2323 } 2324 SET_FIELD(p_ramrod->flags1, 2325 RDMA_REGISTER_TID_RAMROD_DATA_TID_TYPE, tid_type); 2326 2327 p_ramrod->itid = cpu_to_le32(params->itid); 2328 p_ramrod->key = params->key; 2329 p_ramrod->pd = cpu_to_le16(params->pd); 2330 p_ramrod->length_hi = (u8)(params->length >> 32); 2331 p_ramrod->length_lo = DMA_LO_LE(params->length); 2332 if (params->zbva) { 2333 /* Lower 32 bits of the registered MR address. 2334 * In case of zero based MR, will hold FBO 2335 */ 2336 p_ramrod->va.hi = 0; 2337 p_ramrod->va.lo = cpu_to_le32(params->fbo); 2338 } else { 2339 DMA_REGPAIR_LE(p_ramrod->va, params->vaddr); 2340 } 2341 DMA_REGPAIR_LE(p_ramrod->pbl_base, params->pbl_ptr); 2342 2343 /* DIF */ 2344 if (params->dif_enabled) { 2345 SET_FIELD(p_ramrod->flags2, 2346 RDMA_REGISTER_TID_RAMROD_DATA_DIF_ON_HOST_FLG, 1); 2347 DMA_REGPAIR_LE(p_ramrod->dif_error_addr, 2348 params->dif_error_addr); 2349 DMA_REGPAIR_LE(p_ramrod->dif_runt_addr, params->dif_runt_addr); 2350 } 2351 2352 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2353 2354 if (fw_return_code != RDMA_RETURN_OK) { 2355 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2356 return -EINVAL; 2357 } 2358 2359 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Register TID, rc = %d\n", rc); 2360 return rc; 2361 } 2362 2363 static int qed_rdma_deregister_tid(void *rdma_cxt, u32 itid) 2364 { 2365 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2366 struct rdma_deregister_tid_ramrod_data *p_ramrod; 2367 struct qed_sp_init_data init_data; 2368 struct qed_spq_entry *p_ent; 2369 struct qed_ptt *p_ptt; 2370 u8 fw_return_code; 2371 int rc; 2372 2373 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "itid = %08x\n", itid); 2374 2375 /* Get SPQ entry */ 2376 memset(&init_data, 0, sizeof(init_data)); 2377 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2378 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 2379 2380 rc = qed_sp_init_request(p_hwfn, &p_ent, RDMA_RAMROD_DEREGISTER_MR, 2381 p_hwfn->p_rdma_info->proto, &init_data); 2382 if (rc) { 2383 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2384 return rc; 2385 } 2386 2387 p_ramrod = &p_ent->ramrod.rdma_deregister_tid; 2388 p_ramrod->itid = cpu_to_le32(itid); 2389 2390 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2391 if (rc) { 2392 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = %d\n", rc); 2393 return rc; 2394 } 2395 2396 if (fw_return_code == RDMA_RETURN_DEREGISTER_MR_BAD_STATE_ERR) { 2397 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", fw_return_code); 2398 return -EINVAL; 2399 } else if (fw_return_code == RDMA_RETURN_NIG_DRAIN_REQ) { 2400 /* Bit indicating that the TID is in use and a nig drain is 2401 * required before sending the ramrod again 2402 */ 2403 p_ptt = qed_ptt_acquire(p_hwfn); 2404 if (!p_ptt) { 2405 rc = -EBUSY; 2406 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2407 "Failed to acquire PTT\n"); 2408 return rc; 2409 } 2410 2411 rc = qed_mcp_drain(p_hwfn, p_ptt); 2412 if (rc) { 2413 qed_ptt_release(p_hwfn, p_ptt); 2414 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2415 "Drain failed\n"); 2416 return rc; 2417 } 2418 2419 qed_ptt_release(p_hwfn, p_ptt); 2420 2421 /* Resend the ramrod */ 2422 rc = qed_sp_init_request(p_hwfn, &p_ent, 2423 RDMA_RAMROD_DEREGISTER_MR, 2424 p_hwfn->p_rdma_info->proto, 2425 &init_data); 2426 if (rc) { 2427 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2428 "Failed to init sp-element\n"); 2429 return rc; 2430 } 2431 2432 rc = qed_spq_post(p_hwfn, p_ent, &fw_return_code); 2433 if (rc) { 2434 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2435 "Ramrod failed\n"); 2436 return rc; 2437 } 2438 2439 if (fw_return_code != RDMA_RETURN_OK) { 2440 DP_NOTICE(p_hwfn, "fw_return_code = %d\n", 2441 fw_return_code); 2442 return rc; 2443 } 2444 } 2445 2446 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "De-registered TID, rc = %d\n", rc); 2447 return rc; 2448 } 2449 2450 static void *qed_rdma_get_rdma_ctx(struct qed_dev *cdev) 2451 { 2452 return QED_LEADING_HWFN(cdev); 2453 } 2454 2455 static void qed_rdma_dpm_conf(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2456 { 2457 u32 val; 2458 2459 val = (p_hwfn->dcbx_no_edpm || p_hwfn->db_bar_no_edpm) ? 0 : 1; 2460 2461 qed_wr(p_hwfn, p_ptt, DORQ_REG_PF_DPM_ENABLE, val); 2462 DP_VERBOSE(p_hwfn, (QED_MSG_DCB | QED_MSG_RDMA), 2463 "Changing DPM_EN state to %d (DCBX=%d, DB_BAR=%d)\n", 2464 val, p_hwfn->dcbx_no_edpm, p_hwfn->db_bar_no_edpm); 2465 } 2466 2467 void qed_rdma_dpm_bar(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2468 { 2469 p_hwfn->db_bar_no_edpm = true; 2470 2471 qed_rdma_dpm_conf(p_hwfn, p_ptt); 2472 } 2473 2474 static int qed_rdma_start(void *rdma_cxt, 2475 struct qed_rdma_start_in_params *params) 2476 { 2477 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2478 struct qed_ptt *p_ptt; 2479 int rc = -EBUSY; 2480 2481 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2482 "desired_cnq = %08x\n", params->desired_cnq); 2483 2484 p_ptt = qed_ptt_acquire(p_hwfn); 2485 if (!p_ptt) 2486 goto err; 2487 2488 rc = qed_rdma_alloc(p_hwfn, p_ptt, params); 2489 if (rc) 2490 goto err1; 2491 2492 rc = qed_rdma_setup(p_hwfn, p_ptt, params); 2493 if (rc) 2494 goto err2; 2495 2496 qed_ptt_release(p_hwfn, p_ptt); 2497 2498 return rc; 2499 2500 err2: 2501 qed_rdma_free(p_hwfn); 2502 err1: 2503 qed_ptt_release(p_hwfn, p_ptt); 2504 err: 2505 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "RDMA start - error, rc = %d\n", rc); 2506 return rc; 2507 } 2508 2509 static int qed_rdma_init(struct qed_dev *cdev, 2510 struct qed_rdma_start_in_params *params) 2511 { 2512 return qed_rdma_start(QED_LEADING_HWFN(cdev), params); 2513 } 2514 2515 static void qed_rdma_remove_user(void *rdma_cxt, u16 dpi) 2516 { 2517 struct qed_hwfn *p_hwfn = (struct qed_hwfn *)rdma_cxt; 2518 2519 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "dpi = %08x\n", dpi); 2520 2521 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 2522 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->dpi_map, dpi); 2523 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 2524 } 2525 2526 void qed_ll2b_complete_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2527 u8 connection_handle, 2528 void *cookie, 2529 dma_addr_t first_frag_addr, 2530 bool b_last_fragment, bool b_last_packet) 2531 { 2532 struct qed_roce_ll2_packet *packet = cookie; 2533 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2534 2535 roce_ll2->cbs.tx_cb(roce_ll2->cb_cookie, packet); 2536 } 2537 2538 void qed_ll2b_release_tx_gsi_packet(struct qed_hwfn *p_hwfn, 2539 u8 connection_handle, 2540 void *cookie, 2541 dma_addr_t first_frag_addr, 2542 bool b_last_fragment, bool b_last_packet) 2543 { 2544 qed_ll2b_complete_tx_gsi_packet(p_hwfn, connection_handle, 2545 cookie, first_frag_addr, 2546 b_last_fragment, b_last_packet); 2547 } 2548 2549 void qed_ll2b_complete_rx_gsi_packet(struct qed_hwfn *p_hwfn, 2550 u8 connection_handle, 2551 void *cookie, 2552 dma_addr_t rx_buf_addr, 2553 u16 data_length, 2554 u8 data_length_error, 2555 u16 parse_flags, 2556 u16 vlan, 2557 u32 src_mac_addr_hi, 2558 u16 src_mac_addr_lo, bool b_last_packet) 2559 { 2560 struct qed_roce_ll2_info *roce_ll2 = p_hwfn->ll2; 2561 struct qed_roce_ll2_rx_params params; 2562 struct qed_dev *cdev = p_hwfn->cdev; 2563 struct qed_roce_ll2_packet pkt; 2564 2565 DP_VERBOSE(cdev, 2566 QED_MSG_LL2, 2567 "roce ll2 rx complete: bus_addr=%p, len=%d, data_len_err=%d\n", 2568 (void *)(uintptr_t)rx_buf_addr, 2569 data_length, data_length_error); 2570 2571 memset(&pkt, 0, sizeof(pkt)); 2572 pkt.n_seg = 1; 2573 pkt.payload[0].baddr = rx_buf_addr; 2574 pkt.payload[0].len = data_length; 2575 2576 memset(¶ms, 0, sizeof(params)); 2577 params.vlan_id = vlan; 2578 *((u32 *)¶ms.smac[0]) = ntohl(src_mac_addr_hi); 2579 *((u16 *)¶ms.smac[4]) = ntohs(src_mac_addr_lo); 2580 2581 if (data_length_error) { 2582 DP_ERR(cdev, 2583 "roce ll2 rx complete: data length error %d, length=%d\n", 2584 data_length_error, data_length); 2585 params.rc = -EINVAL; 2586 } 2587 2588 roce_ll2->cbs.rx_cb(roce_ll2->cb_cookie, &pkt, ¶ms); 2589 } 2590 2591 static int qed_roce_ll2_set_mac_filter(struct qed_dev *cdev, 2592 u8 *old_mac_address, 2593 u8 *new_mac_address) 2594 { 2595 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2596 struct qed_ptt *p_ptt; 2597 int rc = 0; 2598 2599 if (!hwfn->ll2 || hwfn->ll2->handle == QED_LL2_UNUSED_HANDLE) { 2600 DP_ERR(cdev, 2601 "qed roce mac filter failed - roce_info/ll2 NULL\n"); 2602 return -EINVAL; 2603 } 2604 2605 p_ptt = qed_ptt_acquire(QED_LEADING_HWFN(cdev)); 2606 if (!p_ptt) { 2607 DP_ERR(cdev, 2608 "qed roce ll2 mac filter set: failed to acquire PTT\n"); 2609 return -EINVAL; 2610 } 2611 2612 mutex_lock(&hwfn->ll2->lock); 2613 if (old_mac_address) 2614 qed_llh_remove_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2615 old_mac_address); 2616 if (new_mac_address) 2617 rc = qed_llh_add_mac_filter(QED_LEADING_HWFN(cdev), p_ptt, 2618 new_mac_address); 2619 mutex_unlock(&hwfn->ll2->lock); 2620 2621 qed_ptt_release(QED_LEADING_HWFN(cdev), p_ptt); 2622 2623 if (rc) 2624 DP_ERR(cdev, 2625 "qed roce ll2 mac filter set: failed to add mac filter\n"); 2626 2627 return rc; 2628 } 2629 2630 static int qed_roce_ll2_start(struct qed_dev *cdev, 2631 struct qed_roce_ll2_params *params) 2632 { 2633 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2634 struct qed_roce_ll2_info *roce_ll2; 2635 struct qed_ll2_info ll2_params; 2636 int rc; 2637 2638 if (!params) { 2639 DP_ERR(cdev, "qed roce ll2 start: failed due to NULL params\n"); 2640 return -EINVAL; 2641 } 2642 if (!params->cbs.tx_cb || !params->cbs.rx_cb) { 2643 DP_ERR(cdev, 2644 "qed roce ll2 start: failed due to NULL tx/rx. tx_cb=%p, rx_cb=%p\n", 2645 params->cbs.tx_cb, params->cbs.rx_cb); 2646 return -EINVAL; 2647 } 2648 if (!is_valid_ether_addr(params->mac_address)) { 2649 DP_ERR(cdev, 2650 "qed roce ll2 start: failed due to invalid Ethernet address %pM\n", 2651 params->mac_address); 2652 return -EINVAL; 2653 } 2654 2655 /* Initialize */ 2656 roce_ll2 = kzalloc(sizeof(*roce_ll2), GFP_ATOMIC); 2657 if (!roce_ll2) { 2658 DP_ERR(cdev, "qed roce ll2 start: failed memory allocation\n"); 2659 return -ENOMEM; 2660 } 2661 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2662 roce_ll2->cbs = params->cbs; 2663 roce_ll2->cb_cookie = params->cb_cookie; 2664 mutex_init(&roce_ll2->lock); 2665 2666 memset(&ll2_params, 0, sizeof(ll2_params)); 2667 ll2_params.conn_type = QED_LL2_TYPE_ROCE; 2668 ll2_params.mtu = params->mtu; 2669 ll2_params.rx_drop_ttl0_flg = true; 2670 ll2_params.rx_vlan_removal_en = false; 2671 ll2_params.tx_dest = CORE_TX_DEST_NW; 2672 ll2_params.ai_err_packet_too_big = LL2_DROP_PACKET; 2673 ll2_params.ai_err_no_buf = LL2_DROP_PACKET; 2674 ll2_params.gsi_enable = true; 2675 2676 rc = qed_ll2_acquire_connection(QED_LEADING_HWFN(cdev), &ll2_params, 2677 params->max_rx_buffers, 2678 params->max_tx_buffers, 2679 &roce_ll2->handle); 2680 if (rc) { 2681 DP_ERR(cdev, 2682 "qed roce ll2 start: failed to acquire LL2 connection (rc=%d)\n", 2683 rc); 2684 goto err; 2685 } 2686 2687 rc = qed_ll2_establish_connection(QED_LEADING_HWFN(cdev), 2688 roce_ll2->handle); 2689 if (rc) { 2690 DP_ERR(cdev, 2691 "qed roce ll2 start: failed to establish LL2 connection (rc=%d)\n", 2692 rc); 2693 goto err1; 2694 } 2695 2696 hwfn->ll2 = roce_ll2; 2697 2698 rc = qed_roce_ll2_set_mac_filter(cdev, NULL, params->mac_address); 2699 if (rc) { 2700 hwfn->ll2 = NULL; 2701 goto err2; 2702 } 2703 ether_addr_copy(roce_ll2->mac_address, params->mac_address); 2704 2705 return 0; 2706 2707 err2: 2708 qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2709 err1: 2710 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2711 err: 2712 kfree(roce_ll2); 2713 return rc; 2714 } 2715 2716 static int qed_roce_ll2_stop(struct qed_dev *cdev) 2717 { 2718 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2719 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2720 int rc; 2721 2722 if (roce_ll2->handle == QED_LL2_UNUSED_HANDLE) { 2723 DP_ERR(cdev, "qed roce ll2 stop: cannot stop an unused LL2\n"); 2724 return -EINVAL; 2725 } 2726 2727 /* remove LL2 MAC address filter */ 2728 rc = qed_roce_ll2_set_mac_filter(cdev, roce_ll2->mac_address, NULL); 2729 eth_zero_addr(roce_ll2->mac_address); 2730 2731 rc = qed_ll2_terminate_connection(QED_LEADING_HWFN(cdev), 2732 roce_ll2->handle); 2733 if (rc) 2734 DP_ERR(cdev, 2735 "qed roce ll2 stop: failed to terminate LL2 connection (rc=%d)\n", 2736 rc); 2737 2738 qed_ll2_release_connection(QED_LEADING_HWFN(cdev), roce_ll2->handle); 2739 2740 roce_ll2->handle = QED_LL2_UNUSED_HANDLE; 2741 2742 kfree(roce_ll2); 2743 2744 return rc; 2745 } 2746 2747 static int qed_roce_ll2_tx(struct qed_dev *cdev, 2748 struct qed_roce_ll2_packet *pkt, 2749 struct qed_roce_ll2_tx_params *params) 2750 { 2751 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2752 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2753 enum qed_ll2_roce_flavor_type qed_roce_flavor; 2754 u8 flags = 0; 2755 int rc; 2756 int i; 2757 2758 if (!pkt || !params) { 2759 DP_ERR(cdev, 2760 "roce ll2 tx: failed tx because one of the following is NULL - drv=%p, pkt=%p, params=%p\n", 2761 cdev, pkt, params); 2762 return -EINVAL; 2763 } 2764 2765 qed_roce_flavor = (pkt->roce_mode == ROCE_V1) ? QED_LL2_ROCE 2766 : QED_LL2_RROCE; 2767 2768 if (pkt->roce_mode == ROCE_V2_IPV4) 2769 flags |= BIT(CORE_TX_BD_FLAGS_IP_CSUM_SHIFT); 2770 2771 /* Tx header */ 2772 rc = qed_ll2_prepare_tx_packet(QED_LEADING_HWFN(cdev), roce_ll2->handle, 2773 1 + pkt->n_seg, 0, flags, 0, 2774 QED_LL2_TX_DEST_NW, 2775 qed_roce_flavor, pkt->header.baddr, 2776 pkt->header.len, pkt, 1); 2777 if (rc) { 2778 DP_ERR(cdev, "roce ll2 tx: header failed (rc=%d)\n", rc); 2779 return QED_ROCE_TX_HEAD_FAILURE; 2780 } 2781 2782 /* Tx payload */ 2783 for (i = 0; i < pkt->n_seg; i++) { 2784 rc = qed_ll2_set_fragment_of_tx_packet(QED_LEADING_HWFN(cdev), 2785 roce_ll2->handle, 2786 pkt->payload[i].baddr, 2787 pkt->payload[i].len); 2788 if (rc) { 2789 /* If failed not much to do here, partial packet has 2790 * been posted * we can't free memory, will need to wait 2791 * for completion 2792 */ 2793 DP_ERR(cdev, 2794 "roce ll2 tx: payload failed (rc=%d)\n", rc); 2795 return QED_ROCE_TX_FRAG_FAILURE; 2796 } 2797 } 2798 2799 return 0; 2800 } 2801 2802 static int qed_roce_ll2_post_rx_buffer(struct qed_dev *cdev, 2803 struct qed_roce_ll2_buffer *buf, 2804 u64 cookie, u8 notify_fw) 2805 { 2806 return qed_ll2_post_rx_buffer(QED_LEADING_HWFN(cdev), 2807 QED_LEADING_HWFN(cdev)->ll2->handle, 2808 buf->baddr, buf->len, 2809 (void *)(uintptr_t)cookie, notify_fw); 2810 } 2811 2812 static int qed_roce_ll2_stats(struct qed_dev *cdev, struct qed_ll2_stats *stats) 2813 { 2814 struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); 2815 struct qed_roce_ll2_info *roce_ll2 = hwfn->ll2; 2816 2817 return qed_ll2_get_stats(QED_LEADING_HWFN(cdev), 2818 roce_ll2->handle, stats); 2819 } 2820 2821 static const struct qed_rdma_ops qed_rdma_ops_pass = { 2822 .common = &qed_common_ops_pass, 2823 .fill_dev_info = &qed_fill_rdma_dev_info, 2824 .rdma_get_rdma_ctx = &qed_rdma_get_rdma_ctx, 2825 .rdma_init = &qed_rdma_init, 2826 .rdma_add_user = &qed_rdma_add_user, 2827 .rdma_remove_user = &qed_rdma_remove_user, 2828 .rdma_stop = &qed_rdma_stop, 2829 .rdma_query_port = &qed_rdma_query_port, 2830 .rdma_query_device = &qed_rdma_query_device, 2831 .rdma_get_start_sb = &qed_rdma_get_sb_start, 2832 .rdma_get_rdma_int = &qed_rdma_get_int, 2833 .rdma_set_rdma_int = &qed_rdma_set_int, 2834 .rdma_get_min_cnq_msix = &qed_rdma_get_min_cnq_msix, 2835 .rdma_cnq_prod_update = &qed_rdma_cnq_prod_update, 2836 .rdma_alloc_pd = &qed_rdma_alloc_pd, 2837 .rdma_dealloc_pd = &qed_rdma_free_pd, 2838 .rdma_create_cq = &qed_rdma_create_cq, 2839 .rdma_destroy_cq = &qed_rdma_destroy_cq, 2840 .rdma_create_qp = &qed_rdma_create_qp, 2841 .rdma_modify_qp = &qed_rdma_modify_qp, 2842 .rdma_query_qp = &qed_rdma_query_qp, 2843 .rdma_destroy_qp = &qed_rdma_destroy_qp, 2844 .rdma_alloc_tid = &qed_rdma_alloc_tid, 2845 .rdma_free_tid = &qed_rdma_free_tid, 2846 .rdma_register_tid = &qed_rdma_register_tid, 2847 .rdma_deregister_tid = &qed_rdma_deregister_tid, 2848 .roce_ll2_start = &qed_roce_ll2_start, 2849 .roce_ll2_stop = &qed_roce_ll2_stop, 2850 .roce_ll2_tx = &qed_roce_ll2_tx, 2851 .roce_ll2_post_rx_buffer = &qed_roce_ll2_post_rx_buffer, 2852 .roce_ll2_set_mac_filter = &qed_roce_ll2_set_mac_filter, 2853 .roce_ll2_stats = &qed_roce_ll2_stats, 2854 }; 2855 2856 const struct qed_rdma_ops *qed_get_rdma_ops(void) 2857 { 2858 return &qed_rdma_ops_pass; 2859 } 2860 EXPORT_SYMBOL(qed_get_rdma_ops); 2861