1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/if_ether.h> 33 #include <linux/if_vlan.h> 34 #include <linux/ip.h> 35 #include <linux/ipv6.h> 36 #include <linux/spinlock.h> 37 #include <linux/tcp.h> 38 #include "qed_cxt.h" 39 #include "qed_hw.h" 40 #include "qed_ll2.h" 41 #include "qed_rdma.h" 42 #include "qed_reg_addr.h" 43 #include "qed_sp.h" 44 45 #define QED_IWARP_ORD_DEFAULT 32 46 #define QED_IWARP_IRD_DEFAULT 32 47 #define QED_IWARP_MAX_FW_MSS 4120 48 49 #define QED_EP_SIG 0xecabcdef 50 51 struct mpa_v2_hdr { 52 __be16 ird; 53 __be16 ord; 54 }; 55 56 #define MPA_V2_PEER2PEER_MODEL 0x8000 57 #define MPA_V2_SEND_RTR 0x4000 /* on ird */ 58 #define MPA_V2_READ_RTR 0x4000 /* on ord */ 59 #define MPA_V2_WRITE_RTR 0x8000 60 #define MPA_V2_IRD_ORD_MASK 0x3FFF 61 62 #define MPA_REV2(_mpa_rev) ((_mpa_rev) == MPA_NEGOTIATION_TYPE_ENHANCED) 63 64 #define QED_IWARP_INVALID_TCP_CID 0xffffffff 65 #define QED_IWARP_RCV_WND_SIZE_DEF (256 * 1024) 66 #define QED_IWARP_RCV_WND_SIZE_MIN (64 * 1024) 67 #define TIMESTAMP_HEADER_SIZE (12) 68 69 #define QED_IWARP_TS_EN BIT(0) 70 #define QED_IWARP_DA_EN BIT(1) 71 #define QED_IWARP_PARAM_CRC_NEEDED (1) 72 #define QED_IWARP_PARAM_P2P (1) 73 74 static int qed_iwarp_async_event(struct qed_hwfn *p_hwfn, 75 u8 fw_event_code, u16 echo, 76 union event_ring_data *data, 77 u8 fw_return_code); 78 79 /* Override devinfo with iWARP specific values */ 80 void qed_iwarp_init_devinfo(struct qed_hwfn *p_hwfn) 81 { 82 struct qed_rdma_device *dev = p_hwfn->p_rdma_info->dev; 83 84 dev->max_inline = IWARP_REQ_MAX_INLINE_DATA_SIZE; 85 dev->max_qp = min_t(u32, 86 IWARP_MAX_QPS, 87 p_hwfn->p_rdma_info->num_qps) - 88 QED_IWARP_PREALLOC_CNT; 89 90 dev->max_cq = dev->max_qp; 91 92 dev->max_qp_resp_rd_atomic_resc = QED_IWARP_IRD_DEFAULT; 93 dev->max_qp_req_rd_atomic_resc = QED_IWARP_ORD_DEFAULT; 94 } 95 96 void qed_iwarp_init_hw(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 97 { 98 p_hwfn->rdma_prs_search_reg = PRS_REG_SEARCH_TCP; 99 qed_wr(p_hwfn, p_ptt, p_hwfn->rdma_prs_search_reg, 1); 100 p_hwfn->b_rdma_enabled_in_prs = true; 101 } 102 103 /* We have two cid maps, one for tcp which should be used only from passive 104 * syn processing and replacing a pre-allocated ep in the list. The second 105 * for active tcp and for QPs. 106 */ 107 static void qed_iwarp_cid_cleaned(struct qed_hwfn *p_hwfn, u32 cid) 108 { 109 cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto); 110 111 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 112 113 if (cid < QED_IWARP_PREALLOC_CNT) 114 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 115 cid); 116 else 117 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid); 118 119 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 120 } 121 122 static int qed_iwarp_alloc_cid(struct qed_hwfn *p_hwfn, u32 *cid) 123 { 124 int rc; 125 126 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 127 rc = qed_rdma_bmap_alloc_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, cid); 128 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 129 if (rc) { 130 DP_NOTICE(p_hwfn, "Failed in allocating iwarp cid\n"); 131 return rc; 132 } 133 *cid += qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto); 134 135 rc = qed_cxt_dynamic_ilt_alloc(p_hwfn, QED_ELEM_CXT, *cid); 136 if (rc) 137 qed_iwarp_cid_cleaned(p_hwfn, *cid); 138 139 return rc; 140 } 141 142 static void qed_iwarp_set_tcp_cid(struct qed_hwfn *p_hwfn, u32 cid) 143 { 144 cid -= qed_cxt_get_proto_cid_start(p_hwfn, p_hwfn->p_rdma_info->proto); 145 146 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 147 qed_bmap_set_id(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, cid); 148 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 149 } 150 151 /* This function allocates a cid for passive tcp (called from syn receive) 152 * the reason it's separate from the regular cid allocation is because it 153 * is assured that these cids already have ilt allocated. They are preallocated 154 * to ensure that we won't need to allocate memory during syn processing 155 */ 156 static int qed_iwarp_alloc_tcp_cid(struct qed_hwfn *p_hwfn, u32 *cid) 157 { 158 int rc; 159 160 spin_lock_bh(&p_hwfn->p_rdma_info->lock); 161 162 rc = qed_rdma_bmap_alloc_id(p_hwfn, 163 &p_hwfn->p_rdma_info->tcp_cid_map, cid); 164 165 spin_unlock_bh(&p_hwfn->p_rdma_info->lock); 166 167 if (rc) { 168 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 169 "can't allocate iwarp tcp cid max-count=%d\n", 170 p_hwfn->p_rdma_info->tcp_cid_map.max_count); 171 172 *cid = QED_IWARP_INVALID_TCP_CID; 173 return rc; 174 } 175 176 *cid += qed_cxt_get_proto_cid_start(p_hwfn, 177 p_hwfn->p_rdma_info->proto); 178 return 0; 179 } 180 181 int qed_iwarp_create_qp(struct qed_hwfn *p_hwfn, 182 struct qed_rdma_qp *qp, 183 struct qed_rdma_create_qp_out_params *out_params) 184 { 185 struct iwarp_create_qp_ramrod_data *p_ramrod; 186 struct qed_sp_init_data init_data; 187 struct qed_spq_entry *p_ent; 188 u16 physical_queue; 189 u32 cid; 190 int rc; 191 192 qp->shared_queue = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 193 IWARP_SHARED_QUEUE_PAGE_SIZE, 194 &qp->shared_queue_phys_addr, 195 GFP_KERNEL); 196 if (!qp->shared_queue) 197 return -ENOMEM; 198 199 out_params->sq_pbl_virt = (u8 *)qp->shared_queue + 200 IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET; 201 out_params->sq_pbl_phys = qp->shared_queue_phys_addr + 202 IWARP_SHARED_QUEUE_PAGE_SQ_PBL_OFFSET; 203 out_params->rq_pbl_virt = (u8 *)qp->shared_queue + 204 IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET; 205 out_params->rq_pbl_phys = qp->shared_queue_phys_addr + 206 IWARP_SHARED_QUEUE_PAGE_RQ_PBL_OFFSET; 207 208 rc = qed_iwarp_alloc_cid(p_hwfn, &cid); 209 if (rc) 210 goto err1; 211 212 qp->icid = (u16)cid; 213 214 memset(&init_data, 0, sizeof(init_data)); 215 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 216 init_data.cid = qp->icid; 217 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 218 219 rc = qed_sp_init_request(p_hwfn, &p_ent, 220 IWARP_RAMROD_CMD_ID_CREATE_QP, 221 PROTOCOLID_IWARP, &init_data); 222 if (rc) 223 goto err2; 224 225 p_ramrod = &p_ent->ramrod.iwarp_create_qp; 226 227 SET_FIELD(p_ramrod->flags, 228 IWARP_CREATE_QP_RAMROD_DATA_FMR_AND_RESERVED_EN, 229 qp->fmr_and_reserved_lkey); 230 231 SET_FIELD(p_ramrod->flags, 232 IWARP_CREATE_QP_RAMROD_DATA_SIGNALED_COMP, qp->signal_all); 233 234 SET_FIELD(p_ramrod->flags, 235 IWARP_CREATE_QP_RAMROD_DATA_RDMA_RD_EN, 236 qp->incoming_rdma_read_en); 237 238 SET_FIELD(p_ramrod->flags, 239 IWARP_CREATE_QP_RAMROD_DATA_RDMA_WR_EN, 240 qp->incoming_rdma_write_en); 241 242 SET_FIELD(p_ramrod->flags, 243 IWARP_CREATE_QP_RAMROD_DATA_ATOMIC_EN, 244 qp->incoming_atomic_en); 245 246 SET_FIELD(p_ramrod->flags, 247 IWARP_CREATE_QP_RAMROD_DATA_SRQ_FLG, qp->use_srq); 248 249 p_ramrod->pd = qp->pd; 250 p_ramrod->sq_num_pages = qp->sq_num_pages; 251 p_ramrod->rq_num_pages = qp->rq_num_pages; 252 253 p_ramrod->qp_handle_for_cqe.hi = cpu_to_le32(qp->qp_handle.hi); 254 p_ramrod->qp_handle_for_cqe.lo = cpu_to_le32(qp->qp_handle.lo); 255 256 p_ramrod->cq_cid_for_sq = 257 cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->sq_cq_id); 258 p_ramrod->cq_cid_for_rq = 259 cpu_to_le32((p_hwfn->hw_info.opaque_fid << 16) | qp->rq_cq_id); 260 261 p_ramrod->dpi = cpu_to_le16(qp->dpi); 262 263 physical_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); 264 p_ramrod->physical_q0 = cpu_to_le16(physical_queue); 265 physical_queue = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); 266 p_ramrod->physical_q1 = cpu_to_le16(physical_queue); 267 268 rc = qed_spq_post(p_hwfn, p_ent, NULL); 269 if (rc) 270 goto err2; 271 272 return rc; 273 274 err2: 275 qed_iwarp_cid_cleaned(p_hwfn, cid); 276 err1: 277 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 278 IWARP_SHARED_QUEUE_PAGE_SIZE, 279 qp->shared_queue, qp->shared_queue_phys_addr); 280 281 return rc; 282 } 283 284 static int qed_iwarp_modify_fw(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 285 { 286 struct iwarp_modify_qp_ramrod_data *p_ramrod; 287 struct qed_sp_init_data init_data; 288 struct qed_spq_entry *p_ent; 289 int rc; 290 291 /* Get SPQ entry */ 292 memset(&init_data, 0, sizeof(init_data)); 293 init_data.cid = qp->icid; 294 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 295 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 296 297 rc = qed_sp_init_request(p_hwfn, &p_ent, 298 IWARP_RAMROD_CMD_ID_MODIFY_QP, 299 p_hwfn->p_rdma_info->proto, &init_data); 300 if (rc) 301 return rc; 302 303 p_ramrod = &p_ent->ramrod.iwarp_modify_qp; 304 SET_FIELD(p_ramrod->flags, IWARP_MODIFY_QP_RAMROD_DATA_STATE_TRANS_EN, 305 0x1); 306 if (qp->iwarp_state == QED_IWARP_QP_STATE_CLOSING) 307 p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_CLOSING; 308 else 309 p_ramrod->transition_to_state = IWARP_MODIFY_QP_STATE_ERROR; 310 311 rc = qed_spq_post(p_hwfn, p_ent, NULL); 312 313 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x)rc=%d\n", qp->icid, rc); 314 315 return rc; 316 } 317 318 enum qed_iwarp_qp_state qed_roce2iwarp_state(enum qed_roce_qp_state state) 319 { 320 switch (state) { 321 case QED_ROCE_QP_STATE_RESET: 322 case QED_ROCE_QP_STATE_INIT: 323 case QED_ROCE_QP_STATE_RTR: 324 return QED_IWARP_QP_STATE_IDLE; 325 case QED_ROCE_QP_STATE_RTS: 326 return QED_IWARP_QP_STATE_RTS; 327 case QED_ROCE_QP_STATE_SQD: 328 return QED_IWARP_QP_STATE_CLOSING; 329 case QED_ROCE_QP_STATE_ERR: 330 return QED_IWARP_QP_STATE_ERROR; 331 case QED_ROCE_QP_STATE_SQE: 332 return QED_IWARP_QP_STATE_TERMINATE; 333 default: 334 return QED_IWARP_QP_STATE_ERROR; 335 } 336 } 337 338 static enum qed_roce_qp_state 339 qed_iwarp2roce_state(enum qed_iwarp_qp_state state) 340 { 341 switch (state) { 342 case QED_IWARP_QP_STATE_IDLE: 343 return QED_ROCE_QP_STATE_INIT; 344 case QED_IWARP_QP_STATE_RTS: 345 return QED_ROCE_QP_STATE_RTS; 346 case QED_IWARP_QP_STATE_TERMINATE: 347 return QED_ROCE_QP_STATE_SQE; 348 case QED_IWARP_QP_STATE_CLOSING: 349 return QED_ROCE_QP_STATE_SQD; 350 case QED_IWARP_QP_STATE_ERROR: 351 return QED_ROCE_QP_STATE_ERR; 352 default: 353 return QED_ROCE_QP_STATE_ERR; 354 } 355 } 356 357 const char *iwarp_state_names[] = { 358 "IDLE", 359 "RTS", 360 "TERMINATE", 361 "CLOSING", 362 "ERROR", 363 }; 364 365 int 366 qed_iwarp_modify_qp(struct qed_hwfn *p_hwfn, 367 struct qed_rdma_qp *qp, 368 enum qed_iwarp_qp_state new_state, bool internal) 369 { 370 enum qed_iwarp_qp_state prev_iw_state; 371 bool modify_fw = false; 372 int rc = 0; 373 374 /* modify QP can be called from upper-layer or as a result of async 375 * RST/FIN... therefore need to protect 376 */ 377 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock); 378 prev_iw_state = qp->iwarp_state; 379 380 if (prev_iw_state == new_state) { 381 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock); 382 return 0; 383 } 384 385 switch (prev_iw_state) { 386 case QED_IWARP_QP_STATE_IDLE: 387 switch (new_state) { 388 case QED_IWARP_QP_STATE_RTS: 389 qp->iwarp_state = QED_IWARP_QP_STATE_RTS; 390 break; 391 case QED_IWARP_QP_STATE_ERROR: 392 qp->iwarp_state = QED_IWARP_QP_STATE_ERROR; 393 if (!internal) 394 modify_fw = true; 395 break; 396 default: 397 break; 398 } 399 break; 400 case QED_IWARP_QP_STATE_RTS: 401 switch (new_state) { 402 case QED_IWARP_QP_STATE_CLOSING: 403 if (!internal) 404 modify_fw = true; 405 406 qp->iwarp_state = QED_IWARP_QP_STATE_CLOSING; 407 break; 408 case QED_IWARP_QP_STATE_ERROR: 409 if (!internal) 410 modify_fw = true; 411 qp->iwarp_state = QED_IWARP_QP_STATE_ERROR; 412 break; 413 default: 414 break; 415 } 416 break; 417 case QED_IWARP_QP_STATE_ERROR: 418 switch (new_state) { 419 case QED_IWARP_QP_STATE_IDLE: 420 421 qp->iwarp_state = new_state; 422 break; 423 case QED_IWARP_QP_STATE_CLOSING: 424 /* could happen due to race... do nothing.... */ 425 break; 426 default: 427 rc = -EINVAL; 428 } 429 break; 430 case QED_IWARP_QP_STATE_TERMINATE: 431 case QED_IWARP_QP_STATE_CLOSING: 432 qp->iwarp_state = new_state; 433 break; 434 default: 435 break; 436 } 437 438 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) %s --> %s%s\n", 439 qp->icid, 440 iwarp_state_names[prev_iw_state], 441 iwarp_state_names[qp->iwarp_state], 442 internal ? "internal" : ""); 443 444 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.qp_lock); 445 446 if (modify_fw) 447 rc = qed_iwarp_modify_fw(p_hwfn, qp); 448 449 return rc; 450 } 451 452 int qed_iwarp_fw_destroy(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 453 { 454 struct qed_sp_init_data init_data; 455 struct qed_spq_entry *p_ent; 456 int rc; 457 458 /* Get SPQ entry */ 459 memset(&init_data, 0, sizeof(init_data)); 460 init_data.cid = qp->icid; 461 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 462 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 463 464 rc = qed_sp_init_request(p_hwfn, &p_ent, 465 IWARP_RAMROD_CMD_ID_DESTROY_QP, 466 p_hwfn->p_rdma_info->proto, &init_data); 467 if (rc) 468 return rc; 469 470 rc = qed_spq_post(p_hwfn, p_ent, NULL); 471 472 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) rc = %d\n", qp->icid, rc); 473 474 return rc; 475 } 476 477 static void qed_iwarp_destroy_ep(struct qed_hwfn *p_hwfn, 478 struct qed_iwarp_ep *ep, 479 bool remove_from_active_list) 480 { 481 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 482 sizeof(*ep->ep_buffer_virt), 483 ep->ep_buffer_virt, ep->ep_buffer_phys); 484 485 if (remove_from_active_list) { 486 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 487 list_del(&ep->list_entry); 488 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 489 } 490 491 if (ep->qp) 492 ep->qp->ep = NULL; 493 494 kfree(ep); 495 } 496 497 int qed_iwarp_destroy_qp(struct qed_hwfn *p_hwfn, struct qed_rdma_qp *qp) 498 { 499 struct qed_iwarp_ep *ep = qp->ep; 500 int wait_count = 0; 501 int rc = 0; 502 503 if (qp->iwarp_state != QED_IWARP_QP_STATE_ERROR) { 504 rc = qed_iwarp_modify_qp(p_hwfn, qp, 505 QED_IWARP_QP_STATE_ERROR, false); 506 if (rc) 507 return rc; 508 } 509 510 /* Make sure ep is closed before returning and freeing memory. */ 511 if (ep) { 512 while (ep->state != QED_IWARP_EP_CLOSED && wait_count++ < 200) 513 msleep(100); 514 515 if (ep->state != QED_IWARP_EP_CLOSED) 516 DP_NOTICE(p_hwfn, "ep state close timeout state=%x\n", 517 ep->state); 518 519 qed_iwarp_destroy_ep(p_hwfn, ep, false); 520 } 521 522 rc = qed_iwarp_fw_destroy(p_hwfn, qp); 523 524 if (qp->shared_queue) 525 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 526 IWARP_SHARED_QUEUE_PAGE_SIZE, 527 qp->shared_queue, qp->shared_queue_phys_addr); 528 529 return rc; 530 } 531 532 static int 533 qed_iwarp_create_ep(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep **ep_out) 534 { 535 struct qed_iwarp_ep *ep; 536 int rc; 537 538 ep = kzalloc(sizeof(*ep), GFP_KERNEL); 539 if (!ep) 540 return -ENOMEM; 541 542 ep->state = QED_IWARP_EP_INIT; 543 544 ep->ep_buffer_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 545 sizeof(*ep->ep_buffer_virt), 546 &ep->ep_buffer_phys, 547 GFP_KERNEL); 548 if (!ep->ep_buffer_virt) { 549 rc = -ENOMEM; 550 goto err; 551 } 552 553 ep->sig = QED_EP_SIG; 554 555 *ep_out = ep; 556 557 return 0; 558 559 err: 560 kfree(ep); 561 return rc; 562 } 563 564 static void 565 qed_iwarp_print_tcp_ramrod(struct qed_hwfn *p_hwfn, 566 struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod) 567 { 568 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "local_mac=%x %x %x, remote_mac=%x %x %x\n", 569 p_tcp_ramrod->tcp.local_mac_addr_lo, 570 p_tcp_ramrod->tcp.local_mac_addr_mid, 571 p_tcp_ramrod->tcp.local_mac_addr_hi, 572 p_tcp_ramrod->tcp.remote_mac_addr_lo, 573 p_tcp_ramrod->tcp.remote_mac_addr_mid, 574 p_tcp_ramrod->tcp.remote_mac_addr_hi); 575 576 if (p_tcp_ramrod->tcp.ip_version == TCP_IPV4) { 577 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 578 "local_ip=%pI4h:%x, remote_ip=%pI4h:%x, vlan=%x\n", 579 p_tcp_ramrod->tcp.local_ip, 580 p_tcp_ramrod->tcp.local_port, 581 p_tcp_ramrod->tcp.remote_ip, 582 p_tcp_ramrod->tcp.remote_port, 583 p_tcp_ramrod->tcp.vlan_id); 584 } else { 585 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 586 "local_ip=%pI6:%x, remote_ip=%pI6:%x, vlan=%x\n", 587 p_tcp_ramrod->tcp.local_ip, 588 p_tcp_ramrod->tcp.local_port, 589 p_tcp_ramrod->tcp.remote_ip, 590 p_tcp_ramrod->tcp.remote_port, 591 p_tcp_ramrod->tcp.vlan_id); 592 } 593 594 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 595 "flow_label=%x, ttl=%x, tos_or_tc=%x, mss=%x, rcv_wnd_scale=%x, connect_mode=%x, flags=%x\n", 596 p_tcp_ramrod->tcp.flow_label, 597 p_tcp_ramrod->tcp.ttl, 598 p_tcp_ramrod->tcp.tos_or_tc, 599 p_tcp_ramrod->tcp.mss, 600 p_tcp_ramrod->tcp.rcv_wnd_scale, 601 p_tcp_ramrod->tcp.connect_mode, 602 p_tcp_ramrod->tcp.flags); 603 604 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "syn_ip_payload_length=%x, lo=%x, hi=%x\n", 605 p_tcp_ramrod->tcp.syn_ip_payload_length, 606 p_tcp_ramrod->tcp.syn_phy_addr_lo, 607 p_tcp_ramrod->tcp.syn_phy_addr_hi); 608 } 609 610 static int 611 qed_iwarp_tcp_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 612 { 613 struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; 614 struct iwarp_tcp_offload_ramrod_data *p_tcp_ramrod; 615 struct tcp_offload_params_opt2 *tcp; 616 struct qed_sp_init_data init_data; 617 struct qed_spq_entry *p_ent; 618 dma_addr_t async_output_phys; 619 dma_addr_t in_pdata_phys; 620 u16 physical_q; 621 u8 tcp_flags; 622 int rc; 623 int i; 624 625 memset(&init_data, 0, sizeof(init_data)); 626 init_data.cid = ep->tcp_cid; 627 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 628 if (ep->connect_mode == TCP_CONNECT_PASSIVE) 629 init_data.comp_mode = QED_SPQ_MODE_CB; 630 else 631 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 632 633 rc = qed_sp_init_request(p_hwfn, &p_ent, 634 IWARP_RAMROD_CMD_ID_TCP_OFFLOAD, 635 PROTOCOLID_IWARP, &init_data); 636 if (rc) 637 return rc; 638 639 p_tcp_ramrod = &p_ent->ramrod.iwarp_tcp_offload; 640 641 in_pdata_phys = ep->ep_buffer_phys + 642 offsetof(struct qed_iwarp_ep_memory, in_pdata); 643 DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.incoming_ulp_buffer.addr, 644 in_pdata_phys); 645 646 p_tcp_ramrod->iwarp.incoming_ulp_buffer.len = 647 cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata)); 648 649 async_output_phys = ep->ep_buffer_phys + 650 offsetof(struct qed_iwarp_ep_memory, async_output); 651 DMA_REGPAIR_LE(p_tcp_ramrod->iwarp.async_eqe_output_buf, 652 async_output_phys); 653 654 p_tcp_ramrod->iwarp.handle_for_async.hi = cpu_to_le32(PTR_HI(ep)); 655 p_tcp_ramrod->iwarp.handle_for_async.lo = cpu_to_le32(PTR_LO(ep)); 656 657 physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_OFLD); 658 p_tcp_ramrod->iwarp.physical_q0 = cpu_to_le16(physical_q); 659 physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_ACK); 660 p_tcp_ramrod->iwarp.physical_q1 = cpu_to_le16(physical_q); 661 p_tcp_ramrod->iwarp.mpa_mode = iwarp_info->mpa_rev; 662 663 tcp = &p_tcp_ramrod->tcp; 664 qed_set_fw_mac_addr(&tcp->remote_mac_addr_hi, 665 &tcp->remote_mac_addr_mid, 666 &tcp->remote_mac_addr_lo, ep->remote_mac_addr); 667 qed_set_fw_mac_addr(&tcp->local_mac_addr_hi, &tcp->local_mac_addr_mid, 668 &tcp->local_mac_addr_lo, ep->local_mac_addr); 669 670 tcp->vlan_id = cpu_to_le16(ep->cm_info.vlan); 671 672 tcp_flags = p_hwfn->p_rdma_info->iwarp.tcp_flags; 673 tcp->flags = 0; 674 SET_FIELD(tcp->flags, TCP_OFFLOAD_PARAMS_OPT2_TS_EN, 675 !!(tcp_flags & QED_IWARP_TS_EN)); 676 677 SET_FIELD(tcp->flags, TCP_OFFLOAD_PARAMS_OPT2_DA_EN, 678 !!(tcp_flags & QED_IWARP_DA_EN)); 679 680 tcp->ip_version = ep->cm_info.ip_version; 681 682 for (i = 0; i < 4; i++) { 683 tcp->remote_ip[i] = cpu_to_le32(ep->cm_info.remote_ip[i]); 684 tcp->local_ip[i] = cpu_to_le32(ep->cm_info.local_ip[i]); 685 } 686 687 tcp->remote_port = cpu_to_le16(ep->cm_info.remote_port); 688 tcp->local_port = cpu_to_le16(ep->cm_info.local_port); 689 tcp->mss = cpu_to_le16(ep->mss); 690 tcp->flow_label = 0; 691 tcp->ttl = 0x40; 692 tcp->tos_or_tc = 0; 693 694 tcp->rcv_wnd_scale = (u8)p_hwfn->p_rdma_info->iwarp.rcv_wnd_scale; 695 tcp->connect_mode = ep->connect_mode; 696 697 if (ep->connect_mode == TCP_CONNECT_PASSIVE) { 698 tcp->syn_ip_payload_length = 699 cpu_to_le16(ep->syn_ip_payload_length); 700 tcp->syn_phy_addr_hi = DMA_HI_LE(ep->syn_phy_addr); 701 tcp->syn_phy_addr_lo = DMA_LO_LE(ep->syn_phy_addr); 702 } 703 704 qed_iwarp_print_tcp_ramrod(p_hwfn, p_tcp_ramrod); 705 706 rc = qed_spq_post(p_hwfn, p_ent, NULL); 707 708 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 709 "EP(0x%x) Offload completed rc=%d\n", ep->tcp_cid, rc); 710 711 return rc; 712 } 713 714 static void 715 qed_iwarp_mpa_received(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 716 { 717 struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; 718 struct qed_iwarp_cm_event_params params; 719 struct mpa_v2_hdr *mpa_v2; 720 union async_output *async_data; 721 u16 mpa_ord, mpa_ird; 722 u8 mpa_hdr_size = 0; 723 u8 mpa_rev; 724 725 async_data = &ep->ep_buffer_virt->async_output; 726 727 mpa_rev = async_data->mpa_request.mpa_handshake_mode; 728 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 729 "private_data_len=%x handshake_mode=%x private_data=(%x)\n", 730 async_data->mpa_request.ulp_data_len, 731 mpa_rev, *((u32 *)(ep->ep_buffer_virt->in_pdata))); 732 733 if (mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) { 734 /* Read ord/ird values from private data buffer */ 735 mpa_v2 = (struct mpa_v2_hdr *)ep->ep_buffer_virt->in_pdata; 736 mpa_hdr_size = sizeof(*mpa_v2); 737 738 mpa_ord = ntohs(mpa_v2->ord); 739 mpa_ird = ntohs(mpa_v2->ird); 740 741 /* Temprary store in cm_info incoming ord/ird requested, later 742 * replace with negotiated value during accept 743 */ 744 ep->cm_info.ord = (u8)min_t(u16, 745 (mpa_ord & MPA_V2_IRD_ORD_MASK), 746 QED_IWARP_ORD_DEFAULT); 747 748 ep->cm_info.ird = (u8)min_t(u16, 749 (mpa_ird & MPA_V2_IRD_ORD_MASK), 750 QED_IWARP_IRD_DEFAULT); 751 752 /* Peer2Peer negotiation */ 753 ep->rtr_type = MPA_RTR_TYPE_NONE; 754 if (mpa_ird & MPA_V2_PEER2PEER_MODEL) { 755 if (mpa_ord & MPA_V2_WRITE_RTR) 756 ep->rtr_type |= MPA_RTR_TYPE_ZERO_WRITE; 757 758 if (mpa_ord & MPA_V2_READ_RTR) 759 ep->rtr_type |= MPA_RTR_TYPE_ZERO_READ; 760 761 if (mpa_ird & MPA_V2_SEND_RTR) 762 ep->rtr_type |= MPA_RTR_TYPE_ZERO_SEND; 763 764 ep->rtr_type &= iwarp_info->rtr_type; 765 766 /* if we're left with no match send our capabilities */ 767 if (ep->rtr_type == MPA_RTR_TYPE_NONE) 768 ep->rtr_type = iwarp_info->rtr_type; 769 } 770 771 ep->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED; 772 } else { 773 ep->cm_info.ord = QED_IWARP_ORD_DEFAULT; 774 ep->cm_info.ird = QED_IWARP_IRD_DEFAULT; 775 ep->mpa_rev = MPA_NEGOTIATION_TYPE_BASIC; 776 } 777 778 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 779 "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x rtr:0x%x ulp_data_len = %x mpa_hdr_size = %x\n", 780 mpa_rev, ep->cm_info.ord, ep->cm_info.ird, ep->rtr_type, 781 async_data->mpa_request.ulp_data_len, mpa_hdr_size); 782 783 /* Strip mpa v2 hdr from private data before sending to upper layer */ 784 ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_hdr_size; 785 786 ep->cm_info.private_data_len = async_data->mpa_request.ulp_data_len - 787 mpa_hdr_size; 788 789 params.event = QED_IWARP_EVENT_MPA_REQUEST; 790 params.cm_info = &ep->cm_info; 791 params.ep_context = ep; 792 params.status = 0; 793 794 ep->state = QED_IWARP_EP_MPA_REQ_RCVD; 795 ep->event_cb(ep->cb_context, ¶ms); 796 } 797 798 static int 799 qed_iwarp_mpa_offload(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 800 { 801 struct iwarp_mpa_offload_ramrod_data *p_mpa_ramrod; 802 struct qed_sp_init_data init_data; 803 dma_addr_t async_output_phys; 804 struct qed_spq_entry *p_ent; 805 dma_addr_t out_pdata_phys; 806 dma_addr_t in_pdata_phys; 807 struct qed_rdma_qp *qp; 808 bool reject; 809 int rc; 810 811 if (!ep) 812 return -EINVAL; 813 814 qp = ep->qp; 815 reject = !qp; 816 817 memset(&init_data, 0, sizeof(init_data)); 818 init_data.cid = reject ? ep->tcp_cid : qp->icid; 819 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 820 821 if (ep->connect_mode == TCP_CONNECT_ACTIVE) 822 init_data.comp_mode = QED_SPQ_MODE_CB; 823 else 824 init_data.comp_mode = QED_SPQ_MODE_EBLOCK; 825 826 rc = qed_sp_init_request(p_hwfn, &p_ent, 827 IWARP_RAMROD_CMD_ID_MPA_OFFLOAD, 828 PROTOCOLID_IWARP, &init_data); 829 if (rc) 830 return rc; 831 832 p_mpa_ramrod = &p_ent->ramrod.iwarp_mpa_offload; 833 out_pdata_phys = ep->ep_buffer_phys + 834 offsetof(struct qed_iwarp_ep_memory, out_pdata); 835 DMA_REGPAIR_LE(p_mpa_ramrod->common.outgoing_ulp_buffer.addr, 836 out_pdata_phys); 837 p_mpa_ramrod->common.outgoing_ulp_buffer.len = 838 ep->cm_info.private_data_len; 839 p_mpa_ramrod->common.crc_needed = p_hwfn->p_rdma_info->iwarp.crc_needed; 840 841 p_mpa_ramrod->common.out_rq.ord = ep->cm_info.ord; 842 p_mpa_ramrod->common.out_rq.ird = ep->cm_info.ird; 843 844 p_mpa_ramrod->tcp_cid = p_hwfn->hw_info.opaque_fid << 16 | ep->tcp_cid; 845 846 in_pdata_phys = ep->ep_buffer_phys + 847 offsetof(struct qed_iwarp_ep_memory, in_pdata); 848 p_mpa_ramrod->tcp_connect_side = ep->connect_mode; 849 DMA_REGPAIR_LE(p_mpa_ramrod->incoming_ulp_buffer.addr, 850 in_pdata_phys); 851 p_mpa_ramrod->incoming_ulp_buffer.len = 852 cpu_to_le16(sizeof(ep->ep_buffer_virt->in_pdata)); 853 async_output_phys = ep->ep_buffer_phys + 854 offsetof(struct qed_iwarp_ep_memory, async_output); 855 DMA_REGPAIR_LE(p_mpa_ramrod->async_eqe_output_buf, 856 async_output_phys); 857 p_mpa_ramrod->handle_for_async.hi = cpu_to_le32(PTR_HI(ep)); 858 p_mpa_ramrod->handle_for_async.lo = cpu_to_le32(PTR_LO(ep)); 859 860 if (!reject) { 861 DMA_REGPAIR_LE(p_mpa_ramrod->shared_queue_addr, 862 qp->shared_queue_phys_addr); 863 p_mpa_ramrod->stats_counter_id = 864 RESC_START(p_hwfn, QED_RDMA_STATS_QUEUE) + qp->stats_queue; 865 } else { 866 p_mpa_ramrod->common.reject = 1; 867 } 868 869 p_mpa_ramrod->mode = ep->mpa_rev; 870 SET_FIELD(p_mpa_ramrod->rtr_pref, 871 IWARP_MPA_OFFLOAD_RAMROD_DATA_RTR_SUPPORTED, ep->rtr_type); 872 873 ep->state = QED_IWARP_EP_MPA_OFFLOADED; 874 rc = qed_spq_post(p_hwfn, p_ent, NULL); 875 if (!reject) 876 ep->cid = qp->icid; /* Now they're migrated. */ 877 878 DP_VERBOSE(p_hwfn, 879 QED_MSG_RDMA, 880 "QP(0x%x) EP(0x%x) MPA Offload rc = %d IRD=0x%x ORD=0x%x rtr_type=%d mpa_rev=%d reject=%d\n", 881 reject ? 0xffff : qp->icid, 882 ep->tcp_cid, 883 rc, 884 ep->cm_info.ird, 885 ep->cm_info.ord, ep->rtr_type, ep->mpa_rev, reject); 886 return rc; 887 } 888 889 static void 890 qed_iwarp_return_ep(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 891 { 892 ep->state = QED_IWARP_EP_INIT; 893 if (ep->qp) 894 ep->qp->ep = NULL; 895 ep->qp = NULL; 896 memset(&ep->cm_info, 0, sizeof(ep->cm_info)); 897 898 if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) { 899 /* We don't care about the return code, it's ok if tcp_cid 900 * remains invalid...in this case we'll defer allocation 901 */ 902 qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid); 903 } 904 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 905 906 list_del(&ep->list_entry); 907 list_add_tail(&ep->list_entry, 908 &p_hwfn->p_rdma_info->iwarp.ep_free_list); 909 910 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 911 } 912 913 void 914 qed_iwarp_parse_private_data(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 915 { 916 struct mpa_v2_hdr *mpa_v2_params; 917 union async_output *async_data; 918 u16 mpa_ird, mpa_ord; 919 u8 mpa_data_size = 0; 920 921 if (MPA_REV2(p_hwfn->p_rdma_info->iwarp.mpa_rev)) { 922 mpa_v2_params = 923 (struct mpa_v2_hdr *)(ep->ep_buffer_virt->in_pdata); 924 mpa_data_size = sizeof(*mpa_v2_params); 925 mpa_ird = ntohs(mpa_v2_params->ird); 926 mpa_ord = ntohs(mpa_v2_params->ord); 927 928 ep->cm_info.ird = (u8)(mpa_ord & MPA_V2_IRD_ORD_MASK); 929 ep->cm_info.ord = (u8)(mpa_ird & MPA_V2_IRD_ORD_MASK); 930 } 931 async_data = &ep->ep_buffer_virt->async_output; 932 933 ep->cm_info.private_data = ep->ep_buffer_virt->in_pdata + mpa_data_size; 934 ep->cm_info.private_data_len = async_data->mpa_response.ulp_data_len - 935 mpa_data_size; 936 } 937 938 void 939 qed_iwarp_mpa_reply_arrived(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 940 { 941 struct qed_iwarp_cm_event_params params; 942 943 if (ep->connect_mode == TCP_CONNECT_PASSIVE) { 944 DP_NOTICE(p_hwfn, 945 "MPA reply event not expected on passive side!\n"); 946 return; 947 } 948 949 params.event = QED_IWARP_EVENT_ACTIVE_MPA_REPLY; 950 951 qed_iwarp_parse_private_data(p_hwfn, ep); 952 953 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 954 "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n", 955 ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird); 956 957 params.cm_info = &ep->cm_info; 958 params.ep_context = ep; 959 params.status = 0; 960 961 ep->mpa_reply_processed = true; 962 963 ep->event_cb(ep->cb_context, ¶ms); 964 } 965 966 #define QED_IWARP_CONNECT_MODE_STRING(ep) \ 967 ((ep)->connect_mode == TCP_CONNECT_PASSIVE) ? "Passive" : "Active" 968 969 /* Called as a result of the event: 970 * IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE 971 */ 972 static void 973 qed_iwarp_mpa_complete(struct qed_hwfn *p_hwfn, 974 struct qed_iwarp_ep *ep, u8 fw_return_code) 975 { 976 struct qed_iwarp_cm_event_params params; 977 978 if (ep->connect_mode == TCP_CONNECT_ACTIVE) 979 params.event = QED_IWARP_EVENT_ACTIVE_COMPLETE; 980 else 981 params.event = QED_IWARP_EVENT_PASSIVE_COMPLETE; 982 983 if (ep->connect_mode == TCP_CONNECT_ACTIVE && !ep->mpa_reply_processed) 984 qed_iwarp_parse_private_data(p_hwfn, ep); 985 986 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 987 "MPA_NEGOTIATE (v%d): ORD: 0x%x IRD: 0x%x\n", 988 ep->mpa_rev, ep->cm_info.ord, ep->cm_info.ird); 989 990 params.cm_info = &ep->cm_info; 991 992 params.ep_context = ep; 993 994 ep->state = QED_IWARP_EP_CLOSED; 995 996 switch (fw_return_code) { 997 case RDMA_RETURN_OK: 998 ep->qp->max_rd_atomic_req = ep->cm_info.ord; 999 ep->qp->max_rd_atomic_resp = ep->cm_info.ird; 1000 qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_RTS, 1); 1001 ep->state = QED_IWARP_EP_ESTABLISHED; 1002 params.status = 0; 1003 break; 1004 case IWARP_CONN_ERROR_MPA_TIMEOUT: 1005 DP_NOTICE(p_hwfn, "%s(0x%x) MPA timeout\n", 1006 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1007 params.status = -EBUSY; 1008 break; 1009 case IWARP_CONN_ERROR_MPA_ERROR_REJECT: 1010 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Reject\n", 1011 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1012 params.status = -ECONNREFUSED; 1013 break; 1014 case IWARP_CONN_ERROR_MPA_RST: 1015 DP_NOTICE(p_hwfn, "%s(0x%x) MPA reset(tcp cid: 0x%x)\n", 1016 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid, 1017 ep->tcp_cid); 1018 params.status = -ECONNRESET; 1019 break; 1020 case IWARP_CONN_ERROR_MPA_FIN: 1021 DP_NOTICE(p_hwfn, "%s(0x%x) MPA received FIN\n", 1022 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1023 params.status = -ECONNREFUSED; 1024 break; 1025 case IWARP_CONN_ERROR_MPA_INSUF_IRD: 1026 DP_NOTICE(p_hwfn, "%s(0x%x) MPA insufficient ird\n", 1027 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1028 params.status = -ECONNREFUSED; 1029 break; 1030 case IWARP_CONN_ERROR_MPA_RTR_MISMATCH: 1031 DP_NOTICE(p_hwfn, "%s(0x%x) MPA RTR MISMATCH\n", 1032 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1033 params.status = -ECONNREFUSED; 1034 break; 1035 case IWARP_CONN_ERROR_MPA_INVALID_PACKET: 1036 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n", 1037 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1038 params.status = -ECONNREFUSED; 1039 break; 1040 case IWARP_CONN_ERROR_MPA_LOCAL_ERROR: 1041 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Local Error\n", 1042 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1043 params.status = -ECONNREFUSED; 1044 break; 1045 case IWARP_CONN_ERROR_MPA_TERMINATE: 1046 DP_NOTICE(p_hwfn, "%s(0x%x) MPA TERMINATE\n", 1047 QED_IWARP_CONNECT_MODE_STRING(ep), ep->cid); 1048 params.status = -ECONNREFUSED; 1049 break; 1050 default: 1051 params.status = -ECONNRESET; 1052 break; 1053 } 1054 1055 ep->event_cb(ep->cb_context, ¶ms); 1056 1057 /* on passive side, if there is no associated QP (REJECT) we need to 1058 * return the ep to the pool, (in the regular case we add an element 1059 * in accept instead of this one. 1060 * In both cases we need to remove it from the ep_list. 1061 */ 1062 if (fw_return_code != RDMA_RETURN_OK) { 1063 ep->tcp_cid = QED_IWARP_INVALID_TCP_CID; 1064 if ((ep->connect_mode == TCP_CONNECT_PASSIVE) && 1065 (!ep->qp)) { /* Rejected */ 1066 qed_iwarp_return_ep(p_hwfn, ep); 1067 } else { 1068 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1069 list_del(&ep->list_entry); 1070 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1071 } 1072 } 1073 } 1074 1075 static void 1076 qed_iwarp_mpa_v2_set_private(struct qed_hwfn *p_hwfn, 1077 struct qed_iwarp_ep *ep, u8 *mpa_data_size) 1078 { 1079 struct mpa_v2_hdr *mpa_v2_params; 1080 u16 mpa_ird, mpa_ord; 1081 1082 *mpa_data_size = 0; 1083 if (MPA_REV2(ep->mpa_rev)) { 1084 mpa_v2_params = 1085 (struct mpa_v2_hdr *)ep->ep_buffer_virt->out_pdata; 1086 *mpa_data_size = sizeof(*mpa_v2_params); 1087 1088 mpa_ird = (u16)ep->cm_info.ird; 1089 mpa_ord = (u16)ep->cm_info.ord; 1090 1091 if (ep->rtr_type != MPA_RTR_TYPE_NONE) { 1092 mpa_ird |= MPA_V2_PEER2PEER_MODEL; 1093 1094 if (ep->rtr_type & MPA_RTR_TYPE_ZERO_SEND) 1095 mpa_ird |= MPA_V2_SEND_RTR; 1096 1097 if (ep->rtr_type & MPA_RTR_TYPE_ZERO_WRITE) 1098 mpa_ord |= MPA_V2_WRITE_RTR; 1099 1100 if (ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) 1101 mpa_ord |= MPA_V2_READ_RTR; 1102 } 1103 1104 mpa_v2_params->ird = htons(mpa_ird); 1105 mpa_v2_params->ord = htons(mpa_ord); 1106 1107 DP_VERBOSE(p_hwfn, 1108 QED_MSG_RDMA, 1109 "MPA_NEGOTIATE Header: [%x ord:%x ird] %x ord:%x ird:%x peer2peer:%x rtr_send:%x rtr_write:%x rtr_read:%x\n", 1110 mpa_v2_params->ird, 1111 mpa_v2_params->ord, 1112 *((u32 *)mpa_v2_params), 1113 mpa_ord & MPA_V2_IRD_ORD_MASK, 1114 mpa_ird & MPA_V2_IRD_ORD_MASK, 1115 !!(mpa_ird & MPA_V2_PEER2PEER_MODEL), 1116 !!(mpa_ird & MPA_V2_SEND_RTR), 1117 !!(mpa_ord & MPA_V2_WRITE_RTR), 1118 !!(mpa_ord & MPA_V2_READ_RTR)); 1119 } 1120 } 1121 1122 int qed_iwarp_connect(void *rdma_cxt, 1123 struct qed_iwarp_connect_in *iparams, 1124 struct qed_iwarp_connect_out *oparams) 1125 { 1126 struct qed_hwfn *p_hwfn = rdma_cxt; 1127 struct qed_iwarp_info *iwarp_info; 1128 struct qed_iwarp_ep *ep; 1129 u8 mpa_data_size = 0; 1130 u8 ts_hdr_size = 0; 1131 u32 cid; 1132 int rc; 1133 1134 if ((iparams->cm_info.ord > QED_IWARP_ORD_DEFAULT) || 1135 (iparams->cm_info.ird > QED_IWARP_IRD_DEFAULT)) { 1136 DP_NOTICE(p_hwfn, 1137 "QP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n", 1138 iparams->qp->icid, iparams->cm_info.ord, 1139 iparams->cm_info.ird); 1140 1141 return -EINVAL; 1142 } 1143 1144 iwarp_info = &p_hwfn->p_rdma_info->iwarp; 1145 1146 /* Allocate ep object */ 1147 rc = qed_iwarp_alloc_cid(p_hwfn, &cid); 1148 if (rc) 1149 return rc; 1150 1151 rc = qed_iwarp_create_ep(p_hwfn, &ep); 1152 if (rc) 1153 goto err; 1154 1155 ep->tcp_cid = cid; 1156 1157 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1158 list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list); 1159 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1160 1161 ep->qp = iparams->qp; 1162 ep->qp->ep = ep; 1163 ether_addr_copy(ep->remote_mac_addr, iparams->remote_mac_addr); 1164 ether_addr_copy(ep->local_mac_addr, iparams->local_mac_addr); 1165 memcpy(&ep->cm_info, &iparams->cm_info, sizeof(ep->cm_info)); 1166 1167 ep->cm_info.ord = iparams->cm_info.ord; 1168 ep->cm_info.ird = iparams->cm_info.ird; 1169 1170 ep->rtr_type = iwarp_info->rtr_type; 1171 if (!iwarp_info->peer2peer) 1172 ep->rtr_type = MPA_RTR_TYPE_NONE; 1173 1174 if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && (ep->cm_info.ord == 0)) 1175 ep->cm_info.ord = 1; 1176 1177 ep->mpa_rev = iwarp_info->mpa_rev; 1178 1179 qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); 1180 1181 ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata; 1182 ep->cm_info.private_data_len = iparams->cm_info.private_data_len + 1183 mpa_data_size; 1184 1185 memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, 1186 iparams->cm_info.private_data, 1187 iparams->cm_info.private_data_len); 1188 1189 if (p_hwfn->p_rdma_info->iwarp.tcp_flags & QED_IWARP_TS_EN) 1190 ts_hdr_size = TIMESTAMP_HEADER_SIZE; 1191 1192 ep->mss = iparams->mss - ts_hdr_size; 1193 ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss); 1194 1195 ep->event_cb = iparams->event_cb; 1196 ep->cb_context = iparams->cb_context; 1197 ep->connect_mode = TCP_CONNECT_ACTIVE; 1198 1199 oparams->ep_context = ep; 1200 1201 rc = qed_iwarp_tcp_offload(p_hwfn, ep); 1202 1203 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x) rc = %d\n", 1204 iparams->qp->icid, ep->tcp_cid, rc); 1205 1206 if (rc) { 1207 qed_iwarp_destroy_ep(p_hwfn, ep, true); 1208 goto err; 1209 } 1210 1211 return rc; 1212 err: 1213 qed_iwarp_cid_cleaned(p_hwfn, cid); 1214 1215 return rc; 1216 } 1217 1218 static struct qed_iwarp_ep *qed_iwarp_get_free_ep(struct qed_hwfn *p_hwfn) 1219 { 1220 struct qed_iwarp_ep *ep = NULL; 1221 int rc; 1222 1223 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1224 1225 if (list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) { 1226 DP_ERR(p_hwfn, "Ep list is empty\n"); 1227 goto out; 1228 } 1229 1230 ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list, 1231 struct qed_iwarp_ep, list_entry); 1232 1233 /* in some cases we could have failed allocating a tcp cid when added 1234 * from accept / failure... retry now..this is not the common case. 1235 */ 1236 if (ep->tcp_cid == QED_IWARP_INVALID_TCP_CID) { 1237 rc = qed_iwarp_alloc_tcp_cid(p_hwfn, &ep->tcp_cid); 1238 1239 /* if we fail we could look for another entry with a valid 1240 * tcp_cid, but since we don't expect to reach this anyway 1241 * it's not worth the handling 1242 */ 1243 if (rc) { 1244 ep->tcp_cid = QED_IWARP_INVALID_TCP_CID; 1245 ep = NULL; 1246 goto out; 1247 } 1248 } 1249 1250 list_del(&ep->list_entry); 1251 1252 out: 1253 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1254 return ep; 1255 } 1256 1257 #define QED_IWARP_MAX_CID_CLEAN_TIME 100 1258 #define QED_IWARP_MAX_NO_PROGRESS_CNT 5 1259 1260 /* This function waits for all the bits of a bmap to be cleared, as long as 1261 * there is progress ( i.e. the number of bits left to be cleared decreases ) 1262 * the function continues. 1263 */ 1264 static int 1265 qed_iwarp_wait_cid_map_cleared(struct qed_hwfn *p_hwfn, struct qed_bmap *bmap) 1266 { 1267 int prev_weight = 0; 1268 int wait_count = 0; 1269 int weight = 0; 1270 1271 weight = bitmap_weight(bmap->bitmap, bmap->max_count); 1272 prev_weight = weight; 1273 1274 while (weight) { 1275 msleep(QED_IWARP_MAX_CID_CLEAN_TIME); 1276 1277 weight = bitmap_weight(bmap->bitmap, bmap->max_count); 1278 1279 if (prev_weight == weight) { 1280 wait_count++; 1281 } else { 1282 prev_weight = weight; 1283 wait_count = 0; 1284 } 1285 1286 if (wait_count > QED_IWARP_MAX_NO_PROGRESS_CNT) { 1287 DP_NOTICE(p_hwfn, 1288 "%s bitmap wait timed out (%d cids pending)\n", 1289 bmap->name, weight); 1290 return -EBUSY; 1291 } 1292 } 1293 return 0; 1294 } 1295 1296 static int qed_iwarp_wait_for_all_cids(struct qed_hwfn *p_hwfn) 1297 { 1298 int rc; 1299 int i; 1300 1301 rc = qed_iwarp_wait_cid_map_cleared(p_hwfn, 1302 &p_hwfn->p_rdma_info->tcp_cid_map); 1303 if (rc) 1304 return rc; 1305 1306 /* Now free the tcp cids from the main cid map */ 1307 for (i = 0; i < QED_IWARP_PREALLOC_CNT; i++) 1308 qed_bmap_release_id(p_hwfn, &p_hwfn->p_rdma_info->cid_map, i); 1309 1310 /* Now wait for all cids to be completed */ 1311 return qed_iwarp_wait_cid_map_cleared(p_hwfn, 1312 &p_hwfn->p_rdma_info->cid_map); 1313 } 1314 1315 static void qed_iwarp_free_prealloc_ep(struct qed_hwfn *p_hwfn) 1316 { 1317 struct qed_iwarp_ep *ep; 1318 1319 while (!list_empty(&p_hwfn->p_rdma_info->iwarp.ep_free_list)) { 1320 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1321 1322 ep = list_first_entry(&p_hwfn->p_rdma_info->iwarp.ep_free_list, 1323 struct qed_iwarp_ep, list_entry); 1324 1325 if (!ep) { 1326 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1327 break; 1328 } 1329 list_del(&ep->list_entry); 1330 1331 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1332 1333 if (ep->tcp_cid != QED_IWARP_INVALID_TCP_CID) 1334 qed_iwarp_cid_cleaned(p_hwfn, ep->tcp_cid); 1335 1336 qed_iwarp_destroy_ep(p_hwfn, ep, false); 1337 } 1338 } 1339 1340 static int qed_iwarp_prealloc_ep(struct qed_hwfn *p_hwfn, bool init) 1341 { 1342 struct qed_iwarp_ep *ep; 1343 int rc = 0; 1344 int count; 1345 u32 cid; 1346 int i; 1347 1348 count = init ? QED_IWARP_PREALLOC_CNT : 1; 1349 for (i = 0; i < count; i++) { 1350 rc = qed_iwarp_create_ep(p_hwfn, &ep); 1351 if (rc) 1352 return rc; 1353 1354 /* During initialization we allocate from the main pool, 1355 * afterwards we allocate only from the tcp_cid. 1356 */ 1357 if (init) { 1358 rc = qed_iwarp_alloc_cid(p_hwfn, &cid); 1359 if (rc) 1360 goto err; 1361 qed_iwarp_set_tcp_cid(p_hwfn, cid); 1362 } else { 1363 /* We don't care about the return code, it's ok if 1364 * tcp_cid remains invalid...in this case we'll 1365 * defer allocation 1366 */ 1367 qed_iwarp_alloc_tcp_cid(p_hwfn, &cid); 1368 } 1369 1370 ep->tcp_cid = cid; 1371 1372 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1373 list_add_tail(&ep->list_entry, 1374 &p_hwfn->p_rdma_info->iwarp.ep_free_list); 1375 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1376 } 1377 1378 return rc; 1379 1380 err: 1381 qed_iwarp_destroy_ep(p_hwfn, ep, false); 1382 1383 return rc; 1384 } 1385 1386 int qed_iwarp_alloc(struct qed_hwfn *p_hwfn) 1387 { 1388 int rc; 1389 1390 /* Allocate bitmap for tcp cid. These are used by passive side 1391 * to ensure it can allocate a tcp cid during dpc that was 1392 * pre-acquired and doesn't require dynamic allocation of ilt 1393 */ 1394 rc = qed_rdma_bmap_alloc(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1395 QED_IWARP_PREALLOC_CNT, "TCP_CID"); 1396 if (rc) { 1397 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1398 "Failed to allocate tcp cid, rc = %d\n", rc); 1399 return rc; 1400 } 1401 1402 INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_free_list); 1403 spin_lock_init(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1404 1405 return qed_iwarp_prealloc_ep(p_hwfn, true); 1406 } 1407 1408 void qed_iwarp_resc_free(struct qed_hwfn *p_hwfn) 1409 { 1410 qed_rdma_bmap_free(p_hwfn, &p_hwfn->p_rdma_info->tcp_cid_map, 1); 1411 } 1412 1413 int qed_iwarp_accept(void *rdma_cxt, struct qed_iwarp_accept_in *iparams) 1414 { 1415 struct qed_hwfn *p_hwfn = rdma_cxt; 1416 struct qed_iwarp_ep *ep; 1417 u8 mpa_data_size = 0; 1418 int rc; 1419 1420 ep = iparams->ep_context; 1421 if (!ep) { 1422 DP_ERR(p_hwfn, "Ep Context receive in accept is NULL\n"); 1423 return -EINVAL; 1424 } 1425 1426 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n", 1427 iparams->qp->icid, ep->tcp_cid); 1428 1429 if ((iparams->ord > QED_IWARP_ORD_DEFAULT) || 1430 (iparams->ird > QED_IWARP_IRD_DEFAULT)) { 1431 DP_VERBOSE(p_hwfn, 1432 QED_MSG_RDMA, 1433 "QP(0x%x) EP(0x%x) ERROR: Invalid ord(0x%x)/ird(0x%x)\n", 1434 iparams->qp->icid, 1435 ep->tcp_cid, iparams->ord, iparams->ord); 1436 return -EINVAL; 1437 } 1438 1439 qed_iwarp_prealloc_ep(p_hwfn, false); 1440 1441 ep->cb_context = iparams->cb_context; 1442 ep->qp = iparams->qp; 1443 ep->qp->ep = ep; 1444 1445 if (ep->mpa_rev == MPA_NEGOTIATION_TYPE_ENHANCED) { 1446 /* Negotiate ord/ird: if upperlayer requested ord larger than 1447 * ird advertised by remote, we need to decrease our ord 1448 */ 1449 if (iparams->ord > ep->cm_info.ird) 1450 iparams->ord = ep->cm_info.ird; 1451 1452 if ((ep->rtr_type & MPA_RTR_TYPE_ZERO_READ) && 1453 (iparams->ird == 0)) 1454 iparams->ird = 1; 1455 } 1456 1457 /* Update cm_info ord/ird to be negotiated values */ 1458 ep->cm_info.ord = iparams->ord; 1459 ep->cm_info.ird = iparams->ird; 1460 1461 qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); 1462 1463 ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata; 1464 ep->cm_info.private_data_len = iparams->private_data_len + 1465 mpa_data_size; 1466 1467 memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, 1468 iparams->private_data, iparams->private_data_len); 1469 1470 rc = qed_iwarp_mpa_offload(p_hwfn, ep); 1471 if (rc) 1472 qed_iwarp_modify_qp(p_hwfn, 1473 iparams->qp, QED_IWARP_QP_STATE_ERROR, 1); 1474 1475 return rc; 1476 } 1477 1478 int qed_iwarp_reject(void *rdma_cxt, struct qed_iwarp_reject_in *iparams) 1479 { 1480 struct qed_hwfn *p_hwfn = rdma_cxt; 1481 struct qed_iwarp_ep *ep; 1482 u8 mpa_data_size = 0; 1483 1484 ep = iparams->ep_context; 1485 if (!ep) { 1486 DP_ERR(p_hwfn, "Ep Context receive in reject is NULL\n"); 1487 return -EINVAL; 1488 } 1489 1490 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x)\n", ep->tcp_cid); 1491 1492 ep->cb_context = iparams->cb_context; 1493 ep->qp = NULL; 1494 1495 qed_iwarp_mpa_v2_set_private(p_hwfn, ep, &mpa_data_size); 1496 1497 ep->cm_info.private_data = ep->ep_buffer_virt->out_pdata; 1498 ep->cm_info.private_data_len = iparams->private_data_len + 1499 mpa_data_size; 1500 1501 memcpy((u8 *)ep->ep_buffer_virt->out_pdata + mpa_data_size, 1502 iparams->private_data, iparams->private_data_len); 1503 1504 return qed_iwarp_mpa_offload(p_hwfn, ep); 1505 } 1506 1507 static void 1508 qed_iwarp_print_cm_info(struct qed_hwfn *p_hwfn, 1509 struct qed_iwarp_cm_info *cm_info) 1510 { 1511 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "ip_version = %d\n", 1512 cm_info->ip_version); 1513 1514 if (cm_info->ip_version == QED_TCP_IPV4) 1515 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1516 "remote_ip %pI4h:%x, local_ip %pI4h:%x vlan=%x\n", 1517 cm_info->remote_ip, cm_info->remote_port, 1518 cm_info->local_ip, cm_info->local_port, 1519 cm_info->vlan); 1520 else 1521 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1522 "remote_ip %pI6:%x, local_ip %pI6:%x vlan=%x\n", 1523 cm_info->remote_ip, cm_info->remote_port, 1524 cm_info->local_ip, cm_info->local_port, 1525 cm_info->vlan); 1526 1527 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 1528 "private_data_len = %x ord = %d, ird = %d\n", 1529 cm_info->private_data_len, cm_info->ord, cm_info->ird); 1530 } 1531 1532 static int 1533 qed_iwarp_ll2_post_rx(struct qed_hwfn *p_hwfn, 1534 struct qed_iwarp_ll2_buff *buf, u8 handle) 1535 { 1536 int rc; 1537 1538 rc = qed_ll2_post_rx_buffer(p_hwfn, handle, buf->data_phys_addr, 1539 (u16)buf->buff_size, buf, 1); 1540 if (rc) { 1541 DP_NOTICE(p_hwfn, 1542 "Failed to repost rx buffer to ll2 rc = %d, handle=%d\n", 1543 rc, handle); 1544 dma_free_coherent(&p_hwfn->cdev->pdev->dev, buf->buff_size, 1545 buf->data, buf->data_phys_addr); 1546 kfree(buf); 1547 } 1548 1549 return rc; 1550 } 1551 1552 static bool 1553 qed_iwarp_ep_exists(struct qed_hwfn *p_hwfn, struct qed_iwarp_cm_info *cm_info) 1554 { 1555 struct qed_iwarp_ep *ep = NULL; 1556 bool found = false; 1557 1558 list_for_each_entry(ep, 1559 &p_hwfn->p_rdma_info->iwarp.ep_list, 1560 list_entry) { 1561 if ((ep->cm_info.local_port == cm_info->local_port) && 1562 (ep->cm_info.remote_port == cm_info->remote_port) && 1563 (ep->cm_info.vlan == cm_info->vlan) && 1564 !memcmp(&ep->cm_info.local_ip, cm_info->local_ip, 1565 sizeof(cm_info->local_ip)) && 1566 !memcmp(&ep->cm_info.remote_ip, cm_info->remote_ip, 1567 sizeof(cm_info->remote_ip))) { 1568 found = true; 1569 break; 1570 } 1571 } 1572 1573 if (found) { 1574 DP_NOTICE(p_hwfn, 1575 "SYN received on active connection - dropping\n"); 1576 qed_iwarp_print_cm_info(p_hwfn, cm_info); 1577 1578 return true; 1579 } 1580 1581 return false; 1582 } 1583 1584 static struct qed_iwarp_listener * 1585 qed_iwarp_get_listener(struct qed_hwfn *p_hwfn, 1586 struct qed_iwarp_cm_info *cm_info) 1587 { 1588 struct qed_iwarp_listener *listener = NULL; 1589 static const u32 ip_zero[4] = { 0, 0, 0, 0 }; 1590 bool found = false; 1591 1592 qed_iwarp_print_cm_info(p_hwfn, cm_info); 1593 1594 list_for_each_entry(listener, 1595 &p_hwfn->p_rdma_info->iwarp.listen_list, 1596 list_entry) { 1597 if (listener->port == cm_info->local_port) { 1598 if (!memcmp(listener->ip_addr, 1599 ip_zero, sizeof(ip_zero))) { 1600 found = true; 1601 break; 1602 } 1603 1604 if (!memcmp(listener->ip_addr, 1605 cm_info->local_ip, 1606 sizeof(cm_info->local_ip)) && 1607 (listener->vlan == cm_info->vlan)) { 1608 found = true; 1609 break; 1610 } 1611 } 1612 } 1613 1614 if (found) { 1615 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "listener found = %p\n", 1616 listener); 1617 return listener; 1618 } 1619 1620 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "listener not found\n"); 1621 return NULL; 1622 } 1623 1624 static int 1625 qed_iwarp_parse_rx_pkt(struct qed_hwfn *p_hwfn, 1626 struct qed_iwarp_cm_info *cm_info, 1627 void *buf, 1628 u8 *remote_mac_addr, 1629 u8 *local_mac_addr, 1630 int *payload_len, int *tcp_start_offset) 1631 { 1632 struct vlan_ethhdr *vethh; 1633 bool vlan_valid = false; 1634 struct ipv6hdr *ip6h; 1635 struct ethhdr *ethh; 1636 struct tcphdr *tcph; 1637 struct iphdr *iph; 1638 int eth_hlen; 1639 int ip_hlen; 1640 int eth_type; 1641 int i; 1642 1643 ethh = buf; 1644 eth_type = ntohs(ethh->h_proto); 1645 if (eth_type == ETH_P_8021Q) { 1646 vlan_valid = true; 1647 vethh = (struct vlan_ethhdr *)ethh; 1648 cm_info->vlan = ntohs(vethh->h_vlan_TCI) & VLAN_VID_MASK; 1649 eth_type = ntohs(vethh->h_vlan_encapsulated_proto); 1650 } 1651 1652 eth_hlen = ETH_HLEN + (vlan_valid ? sizeof(u32) : 0); 1653 1654 ether_addr_copy(remote_mac_addr, ethh->h_source); 1655 ether_addr_copy(local_mac_addr, ethh->h_dest); 1656 1657 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "eth_type =%d source mac: %pM\n", 1658 eth_type, ethh->h_source); 1659 1660 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "eth_hlen=%d destination mac: %pM\n", 1661 eth_hlen, ethh->h_dest); 1662 1663 iph = (struct iphdr *)((u8 *)(ethh) + eth_hlen); 1664 1665 if (eth_type == ETH_P_IP) { 1666 cm_info->local_ip[0] = ntohl(iph->daddr); 1667 cm_info->remote_ip[0] = ntohl(iph->saddr); 1668 cm_info->ip_version = TCP_IPV4; 1669 1670 ip_hlen = (iph->ihl) * sizeof(u32); 1671 *payload_len = ntohs(iph->tot_len) - ip_hlen; 1672 } else if (eth_type == ETH_P_IPV6) { 1673 ip6h = (struct ipv6hdr *)iph; 1674 for (i = 0; i < 4; i++) { 1675 cm_info->local_ip[i] = 1676 ntohl(ip6h->daddr.in6_u.u6_addr32[i]); 1677 cm_info->remote_ip[i] = 1678 ntohl(ip6h->saddr.in6_u.u6_addr32[i]); 1679 } 1680 cm_info->ip_version = TCP_IPV6; 1681 1682 ip_hlen = sizeof(*ip6h); 1683 *payload_len = ntohs(ip6h->payload_len); 1684 } else { 1685 DP_NOTICE(p_hwfn, "Unexpected ethertype on ll2 %x\n", eth_type); 1686 return -EINVAL; 1687 } 1688 1689 tcph = (struct tcphdr *)((u8 *)iph + ip_hlen); 1690 1691 if (!tcph->syn) { 1692 DP_NOTICE(p_hwfn, 1693 "Only SYN type packet expected on this ll2 conn, iph->ihl=%d source=%d dest=%d\n", 1694 iph->ihl, tcph->source, tcph->dest); 1695 return -EINVAL; 1696 } 1697 1698 cm_info->local_port = ntohs(tcph->dest); 1699 cm_info->remote_port = ntohs(tcph->source); 1700 1701 qed_iwarp_print_cm_info(p_hwfn, cm_info); 1702 1703 *tcp_start_offset = eth_hlen + ip_hlen; 1704 1705 return 0; 1706 } 1707 1708 static void 1709 qed_iwarp_ll2_comp_syn_pkt(void *cxt, struct qed_ll2_comp_rx_data *data) 1710 { 1711 struct qed_iwarp_ll2_buff *buf = data->cookie; 1712 struct qed_iwarp_listener *listener; 1713 struct qed_ll2_tx_pkt_info tx_pkt; 1714 struct qed_iwarp_cm_info cm_info; 1715 struct qed_hwfn *p_hwfn = cxt; 1716 u8 remote_mac_addr[ETH_ALEN]; 1717 u8 local_mac_addr[ETH_ALEN]; 1718 struct qed_iwarp_ep *ep; 1719 int tcp_start_offset; 1720 u8 ts_hdr_size = 0; 1721 u8 ll2_syn_handle; 1722 int payload_len; 1723 u32 hdr_size; 1724 int rc; 1725 1726 memset(&cm_info, 0, sizeof(cm_info)); 1727 ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle; 1728 if (GET_FIELD(data->parse_flags, 1729 PARSING_AND_ERR_FLAGS_L4CHKSMWASCALCULATED) && 1730 GET_FIELD(data->parse_flags, PARSING_AND_ERR_FLAGS_L4CHKSMERROR)) { 1731 DP_NOTICE(p_hwfn, "Syn packet received with checksum error\n"); 1732 goto err; 1733 } 1734 1735 rc = qed_iwarp_parse_rx_pkt(p_hwfn, &cm_info, (u8 *)(buf->data) + 1736 data->u.placement_offset, remote_mac_addr, 1737 local_mac_addr, &payload_len, 1738 &tcp_start_offset); 1739 if (rc) 1740 goto err; 1741 1742 /* Check if there is a listener for this 4-tuple+vlan */ 1743 listener = qed_iwarp_get_listener(p_hwfn, &cm_info); 1744 if (!listener) { 1745 DP_VERBOSE(p_hwfn, 1746 QED_MSG_RDMA, 1747 "SYN received on tuple not listened on parse_flags=%d packet len=%d\n", 1748 data->parse_flags, data->length.packet_length); 1749 1750 memset(&tx_pkt, 0, sizeof(tx_pkt)); 1751 tx_pkt.num_of_bds = 1; 1752 tx_pkt.vlan = data->vlan; 1753 1754 if (GET_FIELD(data->parse_flags, 1755 PARSING_AND_ERR_FLAGS_TAG8021QEXIST)) 1756 SET_FIELD(tx_pkt.bd_flags, 1757 CORE_TX_BD_DATA_VLAN_INSERTION, 1); 1758 1759 tx_pkt.l4_hdr_offset_w = (data->length.packet_length) >> 2; 1760 tx_pkt.tx_dest = QED_LL2_TX_DEST_LB; 1761 tx_pkt.first_frag = buf->data_phys_addr + 1762 data->u.placement_offset; 1763 tx_pkt.first_frag_len = data->length.packet_length; 1764 tx_pkt.cookie = buf; 1765 1766 rc = qed_ll2_prepare_tx_packet(p_hwfn, ll2_syn_handle, 1767 &tx_pkt, true); 1768 1769 if (rc) { 1770 DP_NOTICE(p_hwfn, 1771 "Can't post SYN back to chip rc=%d\n", rc); 1772 goto err; 1773 } 1774 return; 1775 } 1776 1777 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "Received syn on listening port\n"); 1778 /* There may be an open ep on this connection if this is a syn 1779 * retrasnmit... need to make sure there isn't... 1780 */ 1781 if (qed_iwarp_ep_exists(p_hwfn, &cm_info)) 1782 goto err; 1783 1784 ep = qed_iwarp_get_free_ep(p_hwfn); 1785 if (!ep) 1786 goto err; 1787 1788 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1789 list_add_tail(&ep->list_entry, &p_hwfn->p_rdma_info->iwarp.ep_list); 1790 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 1791 1792 ether_addr_copy(ep->remote_mac_addr, remote_mac_addr); 1793 ether_addr_copy(ep->local_mac_addr, local_mac_addr); 1794 1795 memcpy(&ep->cm_info, &cm_info, sizeof(ep->cm_info)); 1796 1797 if (p_hwfn->p_rdma_info->iwarp.tcp_flags & QED_IWARP_TS_EN) 1798 ts_hdr_size = TIMESTAMP_HEADER_SIZE; 1799 1800 hdr_size = ((cm_info.ip_version == QED_TCP_IPV4) ? 40 : 60) + 1801 ts_hdr_size; 1802 ep->mss = p_hwfn->p_rdma_info->iwarp.max_mtu - hdr_size; 1803 ep->mss = min_t(u16, QED_IWARP_MAX_FW_MSS, ep->mss); 1804 1805 ep->event_cb = listener->event_cb; 1806 ep->cb_context = listener->cb_context; 1807 ep->connect_mode = TCP_CONNECT_PASSIVE; 1808 1809 ep->syn = buf; 1810 ep->syn_ip_payload_length = (u16)payload_len; 1811 ep->syn_phy_addr = buf->data_phys_addr + data->u.placement_offset + 1812 tcp_start_offset; 1813 1814 rc = qed_iwarp_tcp_offload(p_hwfn, ep); 1815 if (rc) { 1816 qed_iwarp_return_ep(p_hwfn, ep); 1817 goto err; 1818 } 1819 1820 return; 1821 err: 1822 qed_iwarp_ll2_post_rx(p_hwfn, buf, ll2_syn_handle); 1823 } 1824 1825 static void qed_iwarp_ll2_rel_rx_pkt(void *cxt, u8 connection_handle, 1826 void *cookie, dma_addr_t rx_buf_addr, 1827 bool b_last_packet) 1828 { 1829 struct qed_iwarp_ll2_buff *buffer = cookie; 1830 struct qed_hwfn *p_hwfn = cxt; 1831 1832 dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size, 1833 buffer->data, buffer->data_phys_addr); 1834 kfree(buffer); 1835 } 1836 1837 static void qed_iwarp_ll2_comp_tx_pkt(void *cxt, u8 connection_handle, 1838 void *cookie, dma_addr_t first_frag_addr, 1839 bool b_last_fragment, bool b_last_packet) 1840 { 1841 struct qed_iwarp_ll2_buff *buffer = cookie; 1842 struct qed_hwfn *p_hwfn = cxt; 1843 1844 /* this was originally an rx packet, post it back */ 1845 qed_iwarp_ll2_post_rx(p_hwfn, buffer, connection_handle); 1846 } 1847 1848 static void qed_iwarp_ll2_rel_tx_pkt(void *cxt, u8 connection_handle, 1849 void *cookie, dma_addr_t first_frag_addr, 1850 bool b_last_fragment, bool b_last_packet) 1851 { 1852 struct qed_iwarp_ll2_buff *buffer = cookie; 1853 struct qed_hwfn *p_hwfn = cxt; 1854 1855 if (!buffer) 1856 return; 1857 1858 dma_free_coherent(&p_hwfn->cdev->pdev->dev, buffer->buff_size, 1859 buffer->data, buffer->data_phys_addr); 1860 1861 kfree(buffer); 1862 } 1863 1864 static int qed_iwarp_ll2_stop(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 1865 { 1866 struct qed_iwarp_info *iwarp_info = &p_hwfn->p_rdma_info->iwarp; 1867 int rc = 0; 1868 1869 if (iwarp_info->ll2_syn_handle != QED_IWARP_HANDLE_INVAL) { 1870 rc = qed_ll2_terminate_connection(p_hwfn, 1871 iwarp_info->ll2_syn_handle); 1872 if (rc) 1873 DP_INFO(p_hwfn, "Failed to terminate syn connection\n"); 1874 1875 qed_ll2_release_connection(p_hwfn, iwarp_info->ll2_syn_handle); 1876 iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL; 1877 } 1878 1879 qed_llh_remove_mac_filter(p_hwfn, 1880 p_ptt, p_hwfn->p_rdma_info->iwarp.mac_addr); 1881 return rc; 1882 } 1883 1884 static int 1885 qed_iwarp_ll2_alloc_buffers(struct qed_hwfn *p_hwfn, 1886 int num_rx_bufs, int buff_size, u8 ll2_handle) 1887 { 1888 struct qed_iwarp_ll2_buff *buffer; 1889 int rc = 0; 1890 int i; 1891 1892 for (i = 0; i < num_rx_bufs; i++) { 1893 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); 1894 if (!buffer) { 1895 rc = -ENOMEM; 1896 break; 1897 } 1898 1899 buffer->data = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 1900 buff_size, 1901 &buffer->data_phys_addr, 1902 GFP_KERNEL); 1903 if (!buffer->data) { 1904 kfree(buffer); 1905 rc = -ENOMEM; 1906 break; 1907 } 1908 1909 buffer->buff_size = buff_size; 1910 rc = qed_iwarp_ll2_post_rx(p_hwfn, buffer, ll2_handle); 1911 if (rc) 1912 /* buffers will be deallocated by qed_ll2 */ 1913 break; 1914 } 1915 return rc; 1916 } 1917 1918 #define QED_IWARP_MAX_BUF_SIZE(mtu) \ 1919 ALIGN((mtu) + ETH_HLEN + 2 * VLAN_HLEN + 2 + ETH_CACHE_LINE_SIZE, \ 1920 ETH_CACHE_LINE_SIZE) 1921 1922 static int 1923 qed_iwarp_ll2_start(struct qed_hwfn *p_hwfn, 1924 struct qed_rdma_start_in_params *params, 1925 struct qed_ptt *p_ptt) 1926 { 1927 struct qed_iwarp_info *iwarp_info; 1928 struct qed_ll2_acquire_data data; 1929 struct qed_ll2_cbs cbs; 1930 int rc = 0; 1931 1932 iwarp_info = &p_hwfn->p_rdma_info->iwarp; 1933 iwarp_info->ll2_syn_handle = QED_IWARP_HANDLE_INVAL; 1934 1935 iwarp_info->max_mtu = params->max_mtu; 1936 1937 ether_addr_copy(p_hwfn->p_rdma_info->iwarp.mac_addr, params->mac_addr); 1938 1939 rc = qed_llh_add_mac_filter(p_hwfn, p_ptt, params->mac_addr); 1940 if (rc) 1941 return rc; 1942 1943 /* Start SYN connection */ 1944 cbs.rx_comp_cb = qed_iwarp_ll2_comp_syn_pkt; 1945 cbs.rx_release_cb = qed_iwarp_ll2_rel_rx_pkt; 1946 cbs.tx_comp_cb = qed_iwarp_ll2_comp_tx_pkt; 1947 cbs.tx_release_cb = qed_iwarp_ll2_rel_tx_pkt; 1948 cbs.cookie = p_hwfn; 1949 1950 memset(&data, 0, sizeof(data)); 1951 data.input.conn_type = QED_LL2_TYPE_IWARP; 1952 data.input.mtu = QED_IWARP_MAX_SYN_PKT_SIZE; 1953 data.input.rx_num_desc = QED_IWARP_LL2_SYN_RX_SIZE; 1954 data.input.tx_num_desc = QED_IWARP_LL2_SYN_TX_SIZE; 1955 data.input.tx_max_bds_per_packet = 1; /* will never be fragmented */ 1956 data.input.tx_tc = PKT_LB_TC; 1957 data.input.tx_dest = QED_LL2_TX_DEST_LB; 1958 data.p_connection_handle = &iwarp_info->ll2_syn_handle; 1959 data.cbs = &cbs; 1960 1961 rc = qed_ll2_acquire_connection(p_hwfn, &data); 1962 if (rc) { 1963 DP_NOTICE(p_hwfn, "Failed to acquire LL2 connection\n"); 1964 qed_llh_remove_mac_filter(p_hwfn, p_ptt, params->mac_addr); 1965 return rc; 1966 } 1967 1968 rc = qed_ll2_establish_connection(p_hwfn, iwarp_info->ll2_syn_handle); 1969 if (rc) { 1970 DP_NOTICE(p_hwfn, "Failed to establish LL2 connection\n"); 1971 goto err; 1972 } 1973 1974 rc = qed_iwarp_ll2_alloc_buffers(p_hwfn, 1975 QED_IWARP_LL2_SYN_RX_SIZE, 1976 QED_IWARP_MAX_SYN_PKT_SIZE, 1977 iwarp_info->ll2_syn_handle); 1978 if (rc) 1979 goto err; 1980 1981 return rc; 1982 err: 1983 qed_iwarp_ll2_stop(p_hwfn, p_ptt); 1984 1985 return rc; 1986 } 1987 1988 int qed_iwarp_setup(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, 1989 struct qed_rdma_start_in_params *params) 1990 { 1991 struct qed_iwarp_info *iwarp_info; 1992 u32 rcv_wnd_size; 1993 1994 iwarp_info = &p_hwfn->p_rdma_info->iwarp; 1995 1996 iwarp_info->tcp_flags = QED_IWARP_TS_EN; 1997 rcv_wnd_size = QED_IWARP_RCV_WND_SIZE_DEF; 1998 1999 /* value 0 is used for ilog2(QED_IWARP_RCV_WND_SIZE_MIN) */ 2000 iwarp_info->rcv_wnd_scale = ilog2(rcv_wnd_size) - 2001 ilog2(QED_IWARP_RCV_WND_SIZE_MIN); 2002 iwarp_info->crc_needed = QED_IWARP_PARAM_CRC_NEEDED; 2003 iwarp_info->mpa_rev = MPA_NEGOTIATION_TYPE_ENHANCED; 2004 2005 iwarp_info->peer2peer = QED_IWARP_PARAM_P2P; 2006 2007 iwarp_info->rtr_type = MPA_RTR_TYPE_ZERO_SEND | 2008 MPA_RTR_TYPE_ZERO_WRITE | 2009 MPA_RTR_TYPE_ZERO_READ; 2010 2011 spin_lock_init(&p_hwfn->p_rdma_info->iwarp.qp_lock); 2012 INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.ep_list); 2013 INIT_LIST_HEAD(&p_hwfn->p_rdma_info->iwarp.listen_list); 2014 2015 qed_spq_register_async_cb(p_hwfn, PROTOCOLID_IWARP, 2016 qed_iwarp_async_event); 2017 2018 return qed_iwarp_ll2_start(p_hwfn, params, p_ptt); 2019 } 2020 2021 int qed_iwarp_stop(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 2022 { 2023 int rc; 2024 2025 qed_iwarp_free_prealloc_ep(p_hwfn); 2026 rc = qed_iwarp_wait_for_all_cids(p_hwfn); 2027 if (rc) 2028 return rc; 2029 2030 qed_spq_unregister_async_cb(p_hwfn, PROTOCOLID_IWARP); 2031 2032 return qed_iwarp_ll2_stop(p_hwfn, p_ptt); 2033 } 2034 2035 void qed_iwarp_qp_in_error(struct qed_hwfn *p_hwfn, 2036 struct qed_iwarp_ep *ep, u8 fw_return_code) 2037 { 2038 struct qed_iwarp_cm_event_params params; 2039 2040 qed_iwarp_modify_qp(p_hwfn, ep->qp, QED_IWARP_QP_STATE_ERROR, true); 2041 2042 params.event = QED_IWARP_EVENT_CLOSE; 2043 params.ep_context = ep; 2044 params.cm_info = &ep->cm_info; 2045 params.status = (fw_return_code == IWARP_QP_IN_ERROR_GOOD_CLOSE) ? 2046 0 : -ECONNRESET; 2047 2048 ep->state = QED_IWARP_EP_CLOSED; 2049 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2050 list_del(&ep->list_entry); 2051 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2052 2053 ep->event_cb(ep->cb_context, ¶ms); 2054 } 2055 2056 void qed_iwarp_exception_received(struct qed_hwfn *p_hwfn, 2057 struct qed_iwarp_ep *ep, int fw_ret_code) 2058 { 2059 struct qed_iwarp_cm_event_params params; 2060 bool event_cb = false; 2061 2062 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "EP(0x%x) fw_ret_code=%d\n", 2063 ep->cid, fw_ret_code); 2064 2065 switch (fw_ret_code) { 2066 case IWARP_EXCEPTION_DETECTED_LLP_CLOSED: 2067 params.status = 0; 2068 params.event = QED_IWARP_EVENT_DISCONNECT; 2069 event_cb = true; 2070 break; 2071 case IWARP_EXCEPTION_DETECTED_LLP_RESET: 2072 params.status = -ECONNRESET; 2073 params.event = QED_IWARP_EVENT_DISCONNECT; 2074 event_cb = true; 2075 break; 2076 case IWARP_EXCEPTION_DETECTED_RQ_EMPTY: 2077 params.event = QED_IWARP_EVENT_RQ_EMPTY; 2078 event_cb = true; 2079 break; 2080 case IWARP_EXCEPTION_DETECTED_IRQ_FULL: 2081 params.event = QED_IWARP_EVENT_IRQ_FULL; 2082 event_cb = true; 2083 break; 2084 case IWARP_EXCEPTION_DETECTED_LLP_TIMEOUT: 2085 params.event = QED_IWARP_EVENT_LLP_TIMEOUT; 2086 event_cb = true; 2087 break; 2088 case IWARP_EXCEPTION_DETECTED_REMOTE_PROTECTION_ERROR: 2089 params.event = QED_IWARP_EVENT_REMOTE_PROTECTION_ERROR; 2090 event_cb = true; 2091 break; 2092 case IWARP_EXCEPTION_DETECTED_CQ_OVERFLOW: 2093 params.event = QED_IWARP_EVENT_CQ_OVERFLOW; 2094 event_cb = true; 2095 break; 2096 case IWARP_EXCEPTION_DETECTED_LOCAL_CATASTROPHIC: 2097 params.event = QED_IWARP_EVENT_QP_CATASTROPHIC; 2098 event_cb = true; 2099 break; 2100 case IWARP_EXCEPTION_DETECTED_LOCAL_ACCESS_ERROR: 2101 params.event = QED_IWARP_EVENT_LOCAL_ACCESS_ERROR; 2102 event_cb = true; 2103 break; 2104 case IWARP_EXCEPTION_DETECTED_REMOTE_OPERATION_ERROR: 2105 params.event = QED_IWARP_EVENT_REMOTE_OPERATION_ERROR; 2106 event_cb = true; 2107 break; 2108 case IWARP_EXCEPTION_DETECTED_TERMINATE_RECEIVED: 2109 params.event = QED_IWARP_EVENT_TERMINATE_RECEIVED; 2110 event_cb = true; 2111 break; 2112 default: 2113 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2114 "Unhandled exception received...fw_ret_code=%d\n", 2115 fw_ret_code); 2116 break; 2117 } 2118 2119 if (event_cb) { 2120 params.ep_context = ep; 2121 params.cm_info = &ep->cm_info; 2122 ep->event_cb(ep->cb_context, ¶ms); 2123 } 2124 } 2125 2126 static void 2127 qed_iwarp_tcp_connect_unsuccessful(struct qed_hwfn *p_hwfn, 2128 struct qed_iwarp_ep *ep, u8 fw_return_code) 2129 { 2130 struct qed_iwarp_cm_event_params params; 2131 2132 memset(¶ms, 0, sizeof(params)); 2133 params.event = QED_IWARP_EVENT_ACTIVE_COMPLETE; 2134 params.ep_context = ep; 2135 params.cm_info = &ep->cm_info; 2136 ep->state = QED_IWARP_EP_CLOSED; 2137 2138 switch (fw_return_code) { 2139 case IWARP_CONN_ERROR_TCP_CONNECT_INVALID_PACKET: 2140 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2141 "%s(0x%x) TCP connect got invalid packet\n", 2142 QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); 2143 params.status = -ECONNRESET; 2144 break; 2145 case IWARP_CONN_ERROR_TCP_CONNECTION_RST: 2146 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2147 "%s(0x%x) TCP Connection Reset\n", 2148 QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); 2149 params.status = -ECONNRESET; 2150 break; 2151 case IWARP_CONN_ERROR_TCP_CONNECT_TIMEOUT: 2152 DP_NOTICE(p_hwfn, "%s(0x%x) TCP timeout\n", 2153 QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); 2154 params.status = -EBUSY; 2155 break; 2156 case IWARP_CONN_ERROR_MPA_NOT_SUPPORTED_VER: 2157 DP_NOTICE(p_hwfn, "%s(0x%x) MPA not supported VER\n", 2158 QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); 2159 params.status = -ECONNREFUSED; 2160 break; 2161 case IWARP_CONN_ERROR_MPA_INVALID_PACKET: 2162 DP_NOTICE(p_hwfn, "%s(0x%x) MPA Invalid Packet\n", 2163 QED_IWARP_CONNECT_MODE_STRING(ep), ep->tcp_cid); 2164 params.status = -ECONNRESET; 2165 break; 2166 default: 2167 DP_ERR(p_hwfn, 2168 "%s(0x%x) Unexpected return code tcp connect: %d\n", 2169 QED_IWARP_CONNECT_MODE_STRING(ep), 2170 ep->tcp_cid, fw_return_code); 2171 params.status = -ECONNRESET; 2172 break; 2173 } 2174 2175 if (ep->connect_mode == TCP_CONNECT_PASSIVE) { 2176 ep->tcp_cid = QED_IWARP_INVALID_TCP_CID; 2177 qed_iwarp_return_ep(p_hwfn, ep); 2178 } else { 2179 ep->event_cb(ep->cb_context, ¶ms); 2180 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2181 list_del(&ep->list_entry); 2182 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2183 } 2184 } 2185 2186 void 2187 qed_iwarp_connect_complete(struct qed_hwfn *p_hwfn, 2188 struct qed_iwarp_ep *ep, u8 fw_return_code) 2189 { 2190 u8 ll2_syn_handle = p_hwfn->p_rdma_info->iwarp.ll2_syn_handle; 2191 2192 if (ep->connect_mode == TCP_CONNECT_PASSIVE) { 2193 /* Done with the SYN packet, post back to ll2 rx */ 2194 qed_iwarp_ll2_post_rx(p_hwfn, ep->syn, ll2_syn_handle); 2195 2196 ep->syn = NULL; 2197 2198 /* If connect failed - upper layer doesn't know about it */ 2199 if (fw_return_code == RDMA_RETURN_OK) 2200 qed_iwarp_mpa_received(p_hwfn, ep); 2201 else 2202 qed_iwarp_tcp_connect_unsuccessful(p_hwfn, ep, 2203 fw_return_code); 2204 } else { 2205 if (fw_return_code == RDMA_RETURN_OK) 2206 qed_iwarp_mpa_offload(p_hwfn, ep); 2207 else 2208 qed_iwarp_tcp_connect_unsuccessful(p_hwfn, ep, 2209 fw_return_code); 2210 } 2211 } 2212 2213 static inline bool 2214 qed_iwarp_check_ep_ok(struct qed_hwfn *p_hwfn, struct qed_iwarp_ep *ep) 2215 { 2216 if (!ep || (ep->sig != QED_EP_SIG)) { 2217 DP_ERR(p_hwfn, "ERROR ON ASYNC ep=%p\n", ep); 2218 return false; 2219 } 2220 2221 return true; 2222 } 2223 2224 static int qed_iwarp_async_event(struct qed_hwfn *p_hwfn, 2225 u8 fw_event_code, u16 echo, 2226 union event_ring_data *data, 2227 u8 fw_return_code) 2228 { 2229 struct regpair *fw_handle = &data->rdma_data.async_handle; 2230 struct qed_iwarp_ep *ep = NULL; 2231 u16 cid; 2232 2233 ep = (struct qed_iwarp_ep *)(uintptr_t)HILO_64(fw_handle->hi, 2234 fw_handle->lo); 2235 2236 switch (fw_event_code) { 2237 case IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE: 2238 /* Async completion after TCP 3-way handshake */ 2239 if (!qed_iwarp_check_ep_ok(p_hwfn, ep)) 2240 return -EINVAL; 2241 DP_VERBOSE(p_hwfn, 2242 QED_MSG_RDMA, 2243 "EP(0x%x) IWARP_EVENT_TYPE_ASYNC_CONNECT_COMPLETE fw_ret_code=%d\n", 2244 ep->tcp_cid, fw_return_code); 2245 qed_iwarp_connect_complete(p_hwfn, ep, fw_return_code); 2246 break; 2247 case IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED: 2248 if (!qed_iwarp_check_ep_ok(p_hwfn, ep)) 2249 return -EINVAL; 2250 DP_VERBOSE(p_hwfn, 2251 QED_MSG_RDMA, 2252 "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_EXCEPTION_DETECTED fw_ret_code=%d\n", 2253 ep->cid, fw_return_code); 2254 qed_iwarp_exception_received(p_hwfn, ep, fw_return_code); 2255 break; 2256 case IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE: 2257 /* Async completion for Close Connection ramrod */ 2258 if (!qed_iwarp_check_ep_ok(p_hwfn, ep)) 2259 return -EINVAL; 2260 DP_VERBOSE(p_hwfn, 2261 QED_MSG_RDMA, 2262 "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_QP_IN_ERROR_STATE fw_ret_code=%d\n", 2263 ep->cid, fw_return_code); 2264 qed_iwarp_qp_in_error(p_hwfn, ep, fw_return_code); 2265 break; 2266 case IWARP_EVENT_TYPE_ASYNC_ENHANCED_MPA_REPLY_ARRIVED: 2267 /* Async event for active side only */ 2268 if (!qed_iwarp_check_ep_ok(p_hwfn, ep)) 2269 return -EINVAL; 2270 DP_VERBOSE(p_hwfn, 2271 QED_MSG_RDMA, 2272 "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_MPA_REPLY_ARRIVED fw_ret_code=%d\n", 2273 ep->cid, fw_return_code); 2274 qed_iwarp_mpa_reply_arrived(p_hwfn, ep); 2275 break; 2276 case IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE: 2277 if (!qed_iwarp_check_ep_ok(p_hwfn, ep)) 2278 return -EINVAL; 2279 DP_VERBOSE(p_hwfn, 2280 QED_MSG_RDMA, 2281 "QP(0x%x) IWARP_EVENT_TYPE_ASYNC_MPA_HANDSHAKE_COMPLETE fw_ret_code=%d\n", 2282 ep->cid, fw_return_code); 2283 qed_iwarp_mpa_complete(p_hwfn, ep, fw_return_code); 2284 break; 2285 case IWARP_EVENT_TYPE_ASYNC_CID_CLEANED: 2286 cid = (u16)le32_to_cpu(fw_handle->lo); 2287 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, 2288 "(0x%x)IWARP_EVENT_TYPE_ASYNC_CID_CLEANED\n", cid); 2289 qed_iwarp_cid_cleaned(p_hwfn, cid); 2290 2291 break; 2292 case IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW: 2293 DP_NOTICE(p_hwfn, "IWARP_EVENT_TYPE_ASYNC_CQ_OVERFLOW\n"); 2294 2295 p_hwfn->p_rdma_info->events.affiliated_event( 2296 p_hwfn->p_rdma_info->events.context, 2297 QED_IWARP_EVENT_CQ_OVERFLOW, 2298 (void *)fw_handle); 2299 break; 2300 default: 2301 DP_ERR(p_hwfn, "Received unexpected async iwarp event %d\n", 2302 fw_event_code); 2303 return -EINVAL; 2304 } 2305 return 0; 2306 } 2307 2308 int 2309 qed_iwarp_create_listen(void *rdma_cxt, 2310 struct qed_iwarp_listen_in *iparams, 2311 struct qed_iwarp_listen_out *oparams) 2312 { 2313 struct qed_hwfn *p_hwfn = rdma_cxt; 2314 struct qed_iwarp_listener *listener; 2315 2316 listener = kzalloc(sizeof(*listener), GFP_KERNEL); 2317 if (!listener) 2318 return -ENOMEM; 2319 2320 listener->ip_version = iparams->ip_version; 2321 memcpy(listener->ip_addr, iparams->ip_addr, sizeof(listener->ip_addr)); 2322 listener->port = iparams->port; 2323 listener->vlan = iparams->vlan; 2324 2325 listener->event_cb = iparams->event_cb; 2326 listener->cb_context = iparams->cb_context; 2327 listener->max_backlog = iparams->max_backlog; 2328 oparams->handle = listener; 2329 2330 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2331 list_add_tail(&listener->list_entry, 2332 &p_hwfn->p_rdma_info->iwarp.listen_list); 2333 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2334 2335 DP_VERBOSE(p_hwfn, 2336 QED_MSG_RDMA, 2337 "callback=%p handle=%p ip=%x:%x:%x:%x port=0x%x vlan=0x%x\n", 2338 listener->event_cb, 2339 listener, 2340 listener->ip_addr[0], 2341 listener->ip_addr[1], 2342 listener->ip_addr[2], 2343 listener->ip_addr[3], listener->port, listener->vlan); 2344 2345 return 0; 2346 } 2347 2348 int qed_iwarp_destroy_listen(void *rdma_cxt, void *handle) 2349 { 2350 struct qed_iwarp_listener *listener = handle; 2351 struct qed_hwfn *p_hwfn = rdma_cxt; 2352 2353 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "handle=%p\n", handle); 2354 2355 spin_lock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2356 list_del(&listener->list_entry); 2357 spin_unlock_bh(&p_hwfn->p_rdma_info->iwarp.iw_lock); 2358 2359 kfree(listener); 2360 2361 return 0; 2362 } 2363 2364 int qed_iwarp_send_rtr(void *rdma_cxt, struct qed_iwarp_send_rtr_in *iparams) 2365 { 2366 struct qed_hwfn *p_hwfn = rdma_cxt; 2367 struct qed_sp_init_data init_data; 2368 struct qed_spq_entry *p_ent; 2369 struct qed_iwarp_ep *ep; 2370 struct qed_rdma_qp *qp; 2371 int rc; 2372 2373 ep = iparams->ep_context; 2374 if (!ep) { 2375 DP_ERR(p_hwfn, "Ep Context receive in send_rtr is NULL\n"); 2376 return -EINVAL; 2377 } 2378 2379 qp = ep->qp; 2380 2381 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "QP(0x%x) EP(0x%x)\n", 2382 qp->icid, ep->tcp_cid); 2383 2384 memset(&init_data, 0, sizeof(init_data)); 2385 init_data.cid = qp->icid; 2386 init_data.opaque_fid = p_hwfn->hw_info.opaque_fid; 2387 init_data.comp_mode = QED_SPQ_MODE_CB; 2388 2389 rc = qed_sp_init_request(p_hwfn, &p_ent, 2390 IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR, 2391 PROTOCOLID_IWARP, &init_data); 2392 2393 if (rc) 2394 return rc; 2395 2396 rc = qed_spq_post(p_hwfn, p_ent, NULL); 2397 2398 DP_VERBOSE(p_hwfn, QED_MSG_RDMA, "rc = 0x%x\n", rc); 2399 2400 return rc; 2401 } 2402 2403 void 2404 qed_iwarp_query_qp(struct qed_rdma_qp *qp, 2405 struct qed_rdma_query_qp_out_params *out_params) 2406 { 2407 out_params->state = qed_iwarp2roce_state(qp->iwarp_state); 2408 } 2409