1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 #include <linux/if_vlan.h> 42 43 #include <net/neighbour.h> 44 #include <net/netevent.h> 45 #include <net/route.h> 46 #include <net/tcp.h> 47 48 #include "iw_cxgb4.h" 49 50 static char *states[] = { 51 "idle", 52 "listen", 53 "connecting", 54 "mpa_wait_req", 55 "mpa_req_sent", 56 "mpa_req_rcvd", 57 "mpa_rep_sent", 58 "fpdu_mode", 59 "aborting", 60 "closing", 61 "moribund", 62 "dead", 63 NULL, 64 }; 65 66 static int nocong; 67 module_param(nocong, int, 0644); 68 MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)"); 69 70 static int enable_ecn; 71 module_param(enable_ecn, int, 0644); 72 MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); 73 74 static int dack_mode = 1; 75 module_param(dack_mode, int, 0644); 76 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 77 78 int c4iw_max_read_depth = 8; 79 module_param(c4iw_max_read_depth, int, 0644); 80 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 81 82 static int enable_tcp_timestamps; 83 module_param(enable_tcp_timestamps, int, 0644); 84 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 85 86 static int enable_tcp_sack; 87 module_param(enable_tcp_sack, int, 0644); 88 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 89 90 static int enable_tcp_window_scaling = 1; 91 module_param(enable_tcp_window_scaling, int, 0644); 92 MODULE_PARM_DESC(enable_tcp_window_scaling, 93 "Enable tcp window scaling (default=1)"); 94 95 int c4iw_debug; 96 module_param(c4iw_debug, int, 0644); 97 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 98 99 static int peer2peer; 100 module_param(peer2peer, int, 0644); 101 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 102 103 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 104 module_param(p2p_type, int, 0644); 105 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 106 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 107 108 static int ep_timeout_secs = 60; 109 module_param(ep_timeout_secs, int, 0644); 110 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 111 "in seconds (default=60)"); 112 113 static int mpa_rev = 1; 114 module_param(mpa_rev, int, 0644); 115 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 116 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 117 " compliant (default=1)"); 118 119 static int markers_enabled; 120 module_param(markers_enabled, int, 0644); 121 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 122 123 static int crc_enabled = 1; 124 module_param(crc_enabled, int, 0644); 125 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 126 127 static int rcv_win = 256 * 1024; 128 module_param(rcv_win, int, 0644); 129 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 130 131 static int snd_win = 128 * 1024; 132 module_param(snd_win, int, 0644); 133 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 134 135 static struct workqueue_struct *workq; 136 137 static struct sk_buff_head rxq; 138 139 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 140 static void ep_timeout(unsigned long arg); 141 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 142 143 static LIST_HEAD(timeout_list); 144 static spinlock_t timeout_lock; 145 146 static void deref_qp(struct c4iw_ep *ep) 147 { 148 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 149 clear_bit(QP_REFERENCED, &ep->com.flags); 150 } 151 152 static void ref_qp(struct c4iw_ep *ep) 153 { 154 set_bit(QP_REFERENCED, &ep->com.flags); 155 c4iw_qp_add_ref(&ep->com.qp->ibqp); 156 } 157 158 static void start_ep_timer(struct c4iw_ep *ep) 159 { 160 PDBG("%s ep %p\n", __func__, ep); 161 if (timer_pending(&ep->timer)) { 162 pr_err("%s timer already started! ep %p\n", 163 __func__, ep); 164 return; 165 } 166 clear_bit(TIMEOUT, &ep->com.flags); 167 c4iw_get_ep(&ep->com); 168 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 169 ep->timer.data = (unsigned long)ep; 170 ep->timer.function = ep_timeout; 171 add_timer(&ep->timer); 172 } 173 174 static void stop_ep_timer(struct c4iw_ep *ep) 175 { 176 PDBG("%s ep %p stopping\n", __func__, ep); 177 del_timer_sync(&ep->timer); 178 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) 179 c4iw_put_ep(&ep->com); 180 } 181 182 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 183 struct l2t_entry *l2e) 184 { 185 int error = 0; 186 187 if (c4iw_fatal_error(rdev)) { 188 kfree_skb(skb); 189 PDBG("%s - device in error state - dropping\n", __func__); 190 return -EIO; 191 } 192 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 193 if (error < 0) 194 kfree_skb(skb); 195 return error < 0 ? error : 0; 196 } 197 198 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 199 { 200 int error = 0; 201 202 if (c4iw_fatal_error(rdev)) { 203 kfree_skb(skb); 204 PDBG("%s - device in error state - dropping\n", __func__); 205 return -EIO; 206 } 207 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 208 if (error < 0) 209 kfree_skb(skb); 210 return error < 0 ? error : 0; 211 } 212 213 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 214 { 215 struct cpl_tid_release *req; 216 217 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 218 if (!skb) 219 return; 220 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 221 INIT_TP_WR(req, hwtid); 222 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 223 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 224 c4iw_ofld_send(rdev, skb); 225 return; 226 } 227 228 static void set_emss(struct c4iw_ep *ep, u16 opt) 229 { 230 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 231 ep->mss = ep->emss; 232 if (GET_TCPOPT_TSTAMP(opt)) 233 ep->emss -= 12; 234 if (ep->emss < 128) 235 ep->emss = 128; 236 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 237 ep->mss, ep->emss); 238 } 239 240 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 241 { 242 enum c4iw_ep_state state; 243 244 mutex_lock(&epc->mutex); 245 state = epc->state; 246 mutex_unlock(&epc->mutex); 247 return state; 248 } 249 250 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 251 { 252 epc->state = new; 253 } 254 255 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 256 { 257 mutex_lock(&epc->mutex); 258 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 259 __state_set(epc, new); 260 mutex_unlock(&epc->mutex); 261 return; 262 } 263 264 static void *alloc_ep(int size, gfp_t gfp) 265 { 266 struct c4iw_ep_common *epc; 267 268 epc = kzalloc(size, gfp); 269 if (epc) { 270 kref_init(&epc->kref); 271 mutex_init(&epc->mutex); 272 c4iw_init_wr_wait(&epc->wr_wait); 273 } 274 PDBG("%s alloc ep %p\n", __func__, epc); 275 return epc; 276 } 277 278 void _c4iw_free_ep(struct kref *kref) 279 { 280 struct c4iw_ep *ep; 281 282 ep = container_of(kref, struct c4iw_ep, com.kref); 283 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 284 if (test_bit(QP_REFERENCED, &ep->com.flags)) 285 deref_qp(ep); 286 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 287 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 288 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 289 dst_release(ep->dst); 290 cxgb4_l2t_release(ep->l2t); 291 } 292 kfree(ep); 293 } 294 295 static void release_ep_resources(struct c4iw_ep *ep) 296 { 297 set_bit(RELEASE_RESOURCES, &ep->com.flags); 298 c4iw_put_ep(&ep->com); 299 } 300 301 static int status2errno(int status) 302 { 303 switch (status) { 304 case CPL_ERR_NONE: 305 return 0; 306 case CPL_ERR_CONN_RESET: 307 return -ECONNRESET; 308 case CPL_ERR_ARP_MISS: 309 return -EHOSTUNREACH; 310 case CPL_ERR_CONN_TIMEDOUT: 311 return -ETIMEDOUT; 312 case CPL_ERR_TCAM_FULL: 313 return -ENOMEM; 314 case CPL_ERR_CONN_EXIST: 315 return -EADDRINUSE; 316 default: 317 return -EIO; 318 } 319 } 320 321 /* 322 * Try and reuse skbs already allocated... 323 */ 324 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 325 { 326 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 327 skb_trim(skb, 0); 328 skb_get(skb); 329 skb_reset_transport_header(skb); 330 } else { 331 skb = alloc_skb(len, gfp); 332 } 333 return skb; 334 } 335 336 static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip, 337 __be32 peer_ip, __be16 local_port, 338 __be16 peer_port, u8 tos) 339 { 340 struct rtable *rt; 341 struct flowi4 fl4; 342 343 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 344 peer_port, local_port, IPPROTO_TCP, 345 tos, 0); 346 if (IS_ERR(rt)) 347 return NULL; 348 return rt; 349 } 350 351 static void arp_failure_discard(void *handle, struct sk_buff *skb) 352 { 353 PDBG("%s c4iw_dev %p\n", __func__, handle); 354 kfree_skb(skb); 355 } 356 357 /* 358 * Handle an ARP failure for an active open. 359 */ 360 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 361 { 362 printk(KERN_ERR MOD "ARP failure duing connect\n"); 363 kfree_skb(skb); 364 } 365 366 /* 367 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 368 * and send it along. 369 */ 370 static void abort_arp_failure(void *handle, struct sk_buff *skb) 371 { 372 struct c4iw_rdev *rdev = handle; 373 struct cpl_abort_req *req = cplhdr(skb); 374 375 PDBG("%s rdev %p\n", __func__, rdev); 376 req->cmd = CPL_ABORT_NO_RST; 377 c4iw_ofld_send(rdev, skb); 378 } 379 380 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 381 { 382 unsigned int flowclen = 80; 383 struct fw_flowc_wr *flowc; 384 int i; 385 386 skb = get_skb(skb, flowclen, GFP_KERNEL); 387 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 388 389 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 390 FW_FLOWC_WR_NPARAMS(8)); 391 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 392 16)) | FW_WR_FLOWID(ep->hwtid)); 393 394 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 395 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 396 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 397 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 398 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 399 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 400 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 401 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 402 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 403 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 404 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 405 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 406 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 407 flowc->mnemval[6].val = cpu_to_be32(snd_win); 408 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 409 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 410 /* Pad WR to 16 byte boundary */ 411 flowc->mnemval[8].mnemonic = 0; 412 flowc->mnemval[8].val = 0; 413 for (i = 0; i < 9; i++) { 414 flowc->mnemval[i].r4[0] = 0; 415 flowc->mnemval[i].r4[1] = 0; 416 flowc->mnemval[i].r4[2] = 0; 417 } 418 419 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 420 c4iw_ofld_send(&ep->com.dev->rdev, skb); 421 } 422 423 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 424 { 425 struct cpl_close_con_req *req; 426 struct sk_buff *skb; 427 int wrlen = roundup(sizeof *req, 16); 428 429 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 430 skb = get_skb(NULL, wrlen, gfp); 431 if (!skb) { 432 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 433 return -ENOMEM; 434 } 435 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 436 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 437 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 438 memset(req, 0, wrlen); 439 INIT_TP_WR(req, ep->hwtid); 440 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 441 ep->hwtid)); 442 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 443 } 444 445 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 446 { 447 struct cpl_abort_req *req; 448 int wrlen = roundup(sizeof *req, 16); 449 450 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 451 skb = get_skb(skb, wrlen, gfp); 452 if (!skb) { 453 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 454 __func__); 455 return -ENOMEM; 456 } 457 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 458 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 459 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 460 memset(req, 0, wrlen); 461 INIT_TP_WR(req, ep->hwtid); 462 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 463 req->cmd = CPL_ABORT_SEND_RST; 464 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 465 } 466 467 #define VLAN_NONE 0xfff 468 #define FILTER_SEL_VLAN_NONE 0xffff 469 #define FILTER_SEL_WIDTH_P_FC (3+1) /* port uses 3 bits, FCoE one bit */ 470 #define FILTER_SEL_WIDTH_VIN_P_FC \ 471 (6 + 7 + FILTER_SEL_WIDTH_P_FC) /* 6 bits are unused, VF uses 7 bits*/ 472 #define FILTER_SEL_WIDTH_TAG_P_FC \ 473 (3 + FILTER_SEL_WIDTH_VIN_P_FC) /* PF uses 3 bits */ 474 #define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC) 475 476 static unsigned int select_ntuple(struct c4iw_dev *dev, struct dst_entry *dst, 477 struct l2t_entry *l2t) 478 { 479 unsigned int ntuple = 0; 480 u32 viid; 481 482 switch (dev->rdev.lldi.filt_mode) { 483 484 /* default filter mode */ 485 case HW_TPL_FR_MT_PR_IV_P_FC: 486 if (l2t->vlan == VLAN_NONE) 487 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 488 else { 489 ntuple |= l2t->vlan << FILTER_SEL_WIDTH_P_FC; 490 ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 491 } 492 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 493 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 494 break; 495 case HW_TPL_FR_MT_PR_OV_P_FC: { 496 viid = cxgb4_port_viid(l2t->neigh->dev); 497 498 ntuple |= FW_VIID_VIN_GET(viid) << FILTER_SEL_WIDTH_P_FC; 499 ntuple |= FW_VIID_PFN_GET(viid) << FILTER_SEL_WIDTH_VIN_P_FC; 500 ntuple |= FW_VIID_VIVLD_GET(viid) << FILTER_SEL_WIDTH_TAG_P_FC; 501 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 502 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 503 break; 504 } 505 default: 506 break; 507 } 508 return ntuple; 509 } 510 511 static int send_connect(struct c4iw_ep *ep) 512 { 513 struct cpl_act_open_req *req; 514 struct sk_buff *skb; 515 u64 opt0; 516 u32 opt2; 517 unsigned int mtu_idx; 518 int wscale; 519 int wrlen = roundup(sizeof *req, 16); 520 521 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 522 523 skb = get_skb(NULL, wrlen, GFP_KERNEL); 524 if (!skb) { 525 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 526 __func__); 527 return -ENOMEM; 528 } 529 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 530 531 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 532 wscale = compute_wscale(rcv_win); 533 opt0 = (nocong ? NO_CONG(1) : 0) | 534 KEEP_ALIVE(1) | 535 DELACK(1) | 536 WND_SCALE(wscale) | 537 MSS_IDX(mtu_idx) | 538 L2T_IDX(ep->l2t->idx) | 539 TX_CHAN(ep->tx_chan) | 540 SMAC_SEL(ep->smac_idx) | 541 DSCP(ep->tos) | 542 ULP_MODE(ULP_MODE_TCPDDP) | 543 RCV_BUFSIZ(rcv_win>>10); 544 opt2 = RX_CHANNEL(0) | 545 CCTRL_ECN(enable_ecn) | 546 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 547 if (enable_tcp_timestamps) 548 opt2 |= TSTAMPS_EN(1); 549 if (enable_tcp_sack) 550 opt2 |= SACK_EN(1); 551 if (wscale && enable_tcp_window_scaling) 552 opt2 |= WND_SCALE_EN(1); 553 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 554 555 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 556 INIT_TP_WR(req, 0); 557 OPCODE_TID(req) = cpu_to_be32( 558 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid))); 559 req->local_port = ep->com.local_addr.sin_port; 560 req->peer_port = ep->com.remote_addr.sin_port; 561 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 562 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 563 req->opt0 = cpu_to_be64(opt0); 564 req->params = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, ep->l2t)); 565 req->opt2 = cpu_to_be32(opt2); 566 set_bit(ACT_OPEN_REQ, &ep->com.history); 567 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 568 } 569 570 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 571 u8 mpa_rev_to_use) 572 { 573 int mpalen, wrlen; 574 struct fw_ofld_tx_data_wr *req; 575 struct mpa_message *mpa; 576 struct mpa_v2_conn_params mpa_v2_params; 577 578 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 579 580 BUG_ON(skb_cloned(skb)); 581 582 mpalen = sizeof(*mpa) + ep->plen; 583 if (mpa_rev_to_use == 2) 584 mpalen += sizeof(struct mpa_v2_conn_params); 585 wrlen = roundup(mpalen + sizeof *req, 16); 586 skb = get_skb(skb, wrlen, GFP_KERNEL); 587 if (!skb) { 588 connect_reply_upcall(ep, -ENOMEM); 589 return; 590 } 591 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 592 593 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 594 memset(req, 0, wrlen); 595 req->op_to_immdlen = cpu_to_be32( 596 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 597 FW_WR_COMPL(1) | 598 FW_WR_IMMDLEN(mpalen)); 599 req->flowid_len16 = cpu_to_be32( 600 FW_WR_FLOWID(ep->hwtid) | 601 FW_WR_LEN16(wrlen >> 4)); 602 req->plen = cpu_to_be32(mpalen); 603 req->tunnel_to_proxy = cpu_to_be32( 604 FW_OFLD_TX_DATA_WR_FLUSH(1) | 605 FW_OFLD_TX_DATA_WR_SHOVE(1)); 606 607 mpa = (struct mpa_message *)(req + 1); 608 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 609 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 610 (markers_enabled ? MPA_MARKERS : 0) | 611 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 612 mpa->private_data_size = htons(ep->plen); 613 mpa->revision = mpa_rev_to_use; 614 if (mpa_rev_to_use == 1) { 615 ep->tried_with_mpa_v1 = 1; 616 ep->retry_with_mpa_v1 = 0; 617 } 618 619 if (mpa_rev_to_use == 2) { 620 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 621 sizeof (struct mpa_v2_conn_params)); 622 mpa_v2_params.ird = htons((u16)ep->ird); 623 mpa_v2_params.ord = htons((u16)ep->ord); 624 625 if (peer2peer) { 626 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 627 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 628 mpa_v2_params.ord |= 629 htons(MPA_V2_RDMA_WRITE_RTR); 630 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 631 mpa_v2_params.ord |= 632 htons(MPA_V2_RDMA_READ_RTR); 633 } 634 memcpy(mpa->private_data, &mpa_v2_params, 635 sizeof(struct mpa_v2_conn_params)); 636 637 if (ep->plen) 638 memcpy(mpa->private_data + 639 sizeof(struct mpa_v2_conn_params), 640 ep->mpa_pkt + sizeof(*mpa), ep->plen); 641 } else 642 if (ep->plen) 643 memcpy(mpa->private_data, 644 ep->mpa_pkt + sizeof(*mpa), ep->plen); 645 646 /* 647 * Reference the mpa skb. This ensures the data area 648 * will remain in memory until the hw acks the tx. 649 * Function fw4_ack() will deref it. 650 */ 651 skb_get(skb); 652 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 653 BUG_ON(ep->mpa_skb); 654 ep->mpa_skb = skb; 655 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 656 start_ep_timer(ep); 657 state_set(&ep->com, MPA_REQ_SENT); 658 ep->mpa_attr.initiator = 1; 659 return; 660 } 661 662 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 663 { 664 int mpalen, wrlen; 665 struct fw_ofld_tx_data_wr *req; 666 struct mpa_message *mpa; 667 struct sk_buff *skb; 668 struct mpa_v2_conn_params mpa_v2_params; 669 670 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 671 672 mpalen = sizeof(*mpa) + plen; 673 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 674 mpalen += sizeof(struct mpa_v2_conn_params); 675 wrlen = roundup(mpalen + sizeof *req, 16); 676 677 skb = get_skb(NULL, wrlen, GFP_KERNEL); 678 if (!skb) { 679 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 680 return -ENOMEM; 681 } 682 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 683 684 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 685 memset(req, 0, wrlen); 686 req->op_to_immdlen = cpu_to_be32( 687 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 688 FW_WR_COMPL(1) | 689 FW_WR_IMMDLEN(mpalen)); 690 req->flowid_len16 = cpu_to_be32( 691 FW_WR_FLOWID(ep->hwtid) | 692 FW_WR_LEN16(wrlen >> 4)); 693 req->plen = cpu_to_be32(mpalen); 694 req->tunnel_to_proxy = cpu_to_be32( 695 FW_OFLD_TX_DATA_WR_FLUSH(1) | 696 FW_OFLD_TX_DATA_WR_SHOVE(1)); 697 698 mpa = (struct mpa_message *)(req + 1); 699 memset(mpa, 0, sizeof(*mpa)); 700 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 701 mpa->flags = MPA_REJECT; 702 mpa->revision = ep->mpa_attr.version; 703 mpa->private_data_size = htons(plen); 704 705 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 706 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 707 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 708 sizeof (struct mpa_v2_conn_params)); 709 mpa_v2_params.ird = htons(((u16)ep->ird) | 710 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 711 0)); 712 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 713 (p2p_type == 714 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 715 MPA_V2_RDMA_WRITE_RTR : p2p_type == 716 FW_RI_INIT_P2PTYPE_READ_REQ ? 717 MPA_V2_RDMA_READ_RTR : 0) : 0)); 718 memcpy(mpa->private_data, &mpa_v2_params, 719 sizeof(struct mpa_v2_conn_params)); 720 721 if (ep->plen) 722 memcpy(mpa->private_data + 723 sizeof(struct mpa_v2_conn_params), pdata, plen); 724 } else 725 if (plen) 726 memcpy(mpa->private_data, pdata, plen); 727 728 /* 729 * Reference the mpa skb again. This ensures the data area 730 * will remain in memory until the hw acks the tx. 731 * Function fw4_ack() will deref it. 732 */ 733 skb_get(skb); 734 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 735 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 736 BUG_ON(ep->mpa_skb); 737 ep->mpa_skb = skb; 738 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 739 } 740 741 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 742 { 743 int mpalen, wrlen; 744 struct fw_ofld_tx_data_wr *req; 745 struct mpa_message *mpa; 746 struct sk_buff *skb; 747 struct mpa_v2_conn_params mpa_v2_params; 748 749 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 750 751 mpalen = sizeof(*mpa) + plen; 752 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 753 mpalen += sizeof(struct mpa_v2_conn_params); 754 wrlen = roundup(mpalen + sizeof *req, 16); 755 756 skb = get_skb(NULL, wrlen, GFP_KERNEL); 757 if (!skb) { 758 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 759 return -ENOMEM; 760 } 761 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 762 763 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 764 memset(req, 0, wrlen); 765 req->op_to_immdlen = cpu_to_be32( 766 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 767 FW_WR_COMPL(1) | 768 FW_WR_IMMDLEN(mpalen)); 769 req->flowid_len16 = cpu_to_be32( 770 FW_WR_FLOWID(ep->hwtid) | 771 FW_WR_LEN16(wrlen >> 4)); 772 req->plen = cpu_to_be32(mpalen); 773 req->tunnel_to_proxy = cpu_to_be32( 774 FW_OFLD_TX_DATA_WR_FLUSH(1) | 775 FW_OFLD_TX_DATA_WR_SHOVE(1)); 776 777 mpa = (struct mpa_message *)(req + 1); 778 memset(mpa, 0, sizeof(*mpa)); 779 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 780 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 781 (markers_enabled ? MPA_MARKERS : 0); 782 mpa->revision = ep->mpa_attr.version; 783 mpa->private_data_size = htons(plen); 784 785 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 786 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 787 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 788 sizeof (struct mpa_v2_conn_params)); 789 mpa_v2_params.ird = htons((u16)ep->ird); 790 mpa_v2_params.ord = htons((u16)ep->ord); 791 if (peer2peer && (ep->mpa_attr.p2p_type != 792 FW_RI_INIT_P2PTYPE_DISABLED)) { 793 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 794 795 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 796 mpa_v2_params.ord |= 797 htons(MPA_V2_RDMA_WRITE_RTR); 798 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 799 mpa_v2_params.ord |= 800 htons(MPA_V2_RDMA_READ_RTR); 801 } 802 803 memcpy(mpa->private_data, &mpa_v2_params, 804 sizeof(struct mpa_v2_conn_params)); 805 806 if (ep->plen) 807 memcpy(mpa->private_data + 808 sizeof(struct mpa_v2_conn_params), pdata, plen); 809 } else 810 if (plen) 811 memcpy(mpa->private_data, pdata, plen); 812 813 /* 814 * Reference the mpa skb. This ensures the data area 815 * will remain in memory until the hw acks the tx. 816 * Function fw4_ack() will deref it. 817 */ 818 skb_get(skb); 819 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 820 ep->mpa_skb = skb; 821 state_set(&ep->com, MPA_REP_SENT); 822 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 823 } 824 825 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 826 { 827 struct c4iw_ep *ep; 828 struct cpl_act_establish *req = cplhdr(skb); 829 unsigned int tid = GET_TID(req); 830 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 831 struct tid_info *t = dev->rdev.lldi.tids; 832 833 ep = lookup_atid(t, atid); 834 835 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 836 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 837 838 dst_confirm(ep->dst); 839 840 /* setup the hwtid for this connection */ 841 ep->hwtid = tid; 842 cxgb4_insert_tid(t, ep, tid); 843 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 844 845 ep->snd_seq = be32_to_cpu(req->snd_isn); 846 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 847 848 set_emss(ep, ntohs(req->tcp_opt)); 849 850 /* dealloc the atid */ 851 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 852 cxgb4_free_atid(t, atid); 853 set_bit(ACT_ESTAB, &ep->com.history); 854 855 /* start MPA negotiation */ 856 send_flowc(ep, NULL); 857 if (ep->retry_with_mpa_v1) 858 send_mpa_req(ep, skb, 1); 859 else 860 send_mpa_req(ep, skb, mpa_rev); 861 862 return 0; 863 } 864 865 static void close_complete_upcall(struct c4iw_ep *ep) 866 { 867 struct iw_cm_event event; 868 869 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 870 memset(&event, 0, sizeof(event)); 871 event.event = IW_CM_EVENT_CLOSE; 872 if (ep->com.cm_id) { 873 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 874 ep, ep->com.cm_id, ep->hwtid); 875 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 876 ep->com.cm_id->rem_ref(ep->com.cm_id); 877 ep->com.cm_id = NULL; 878 set_bit(CLOSE_UPCALL, &ep->com.history); 879 } 880 } 881 882 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 883 { 884 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 885 close_complete_upcall(ep); 886 state_set(&ep->com, ABORTING); 887 set_bit(ABORT_CONN, &ep->com.history); 888 return send_abort(ep, skb, gfp); 889 } 890 891 static void peer_close_upcall(struct c4iw_ep *ep) 892 { 893 struct iw_cm_event event; 894 895 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 896 memset(&event, 0, sizeof(event)); 897 event.event = IW_CM_EVENT_DISCONNECT; 898 if (ep->com.cm_id) { 899 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 900 ep, ep->com.cm_id, ep->hwtid); 901 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 902 set_bit(DISCONN_UPCALL, &ep->com.history); 903 } 904 } 905 906 static void peer_abort_upcall(struct c4iw_ep *ep) 907 { 908 struct iw_cm_event event; 909 910 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 911 memset(&event, 0, sizeof(event)); 912 event.event = IW_CM_EVENT_CLOSE; 913 event.status = -ECONNRESET; 914 if (ep->com.cm_id) { 915 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 916 ep->com.cm_id, ep->hwtid); 917 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 918 ep->com.cm_id->rem_ref(ep->com.cm_id); 919 ep->com.cm_id = NULL; 920 set_bit(ABORT_UPCALL, &ep->com.history); 921 } 922 } 923 924 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 925 { 926 struct iw_cm_event event; 927 928 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 929 memset(&event, 0, sizeof(event)); 930 event.event = IW_CM_EVENT_CONNECT_REPLY; 931 event.status = status; 932 event.local_addr = ep->com.local_addr; 933 event.remote_addr = ep->com.remote_addr; 934 935 if ((status == 0) || (status == -ECONNREFUSED)) { 936 if (!ep->tried_with_mpa_v1) { 937 /* this means MPA_v2 is used */ 938 event.private_data_len = ep->plen - 939 sizeof(struct mpa_v2_conn_params); 940 event.private_data = ep->mpa_pkt + 941 sizeof(struct mpa_message) + 942 sizeof(struct mpa_v2_conn_params); 943 } else { 944 /* this means MPA_v1 is used */ 945 event.private_data_len = ep->plen; 946 event.private_data = ep->mpa_pkt + 947 sizeof(struct mpa_message); 948 } 949 } 950 951 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 952 ep->hwtid, status); 953 set_bit(CONN_RPL_UPCALL, &ep->com.history); 954 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 955 956 if (status < 0) { 957 ep->com.cm_id->rem_ref(ep->com.cm_id); 958 ep->com.cm_id = NULL; 959 } 960 } 961 962 static void connect_request_upcall(struct c4iw_ep *ep) 963 { 964 struct iw_cm_event event; 965 966 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 967 memset(&event, 0, sizeof(event)); 968 event.event = IW_CM_EVENT_CONNECT_REQUEST; 969 event.local_addr = ep->com.local_addr; 970 event.remote_addr = ep->com.remote_addr; 971 event.provider_data = ep; 972 if (!ep->tried_with_mpa_v1) { 973 /* this means MPA_v2 is used */ 974 event.ord = ep->ord; 975 event.ird = ep->ird; 976 event.private_data_len = ep->plen - 977 sizeof(struct mpa_v2_conn_params); 978 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 979 sizeof(struct mpa_v2_conn_params); 980 } else { 981 /* this means MPA_v1 is used. Send max supported */ 982 event.ord = c4iw_max_read_depth; 983 event.ird = c4iw_max_read_depth; 984 event.private_data_len = ep->plen; 985 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 986 } 987 if (state_read(&ep->parent_ep->com) != DEAD) { 988 c4iw_get_ep(&ep->com); 989 ep->parent_ep->com.cm_id->event_handler( 990 ep->parent_ep->com.cm_id, 991 &event); 992 } 993 set_bit(CONNREQ_UPCALL, &ep->com.history); 994 c4iw_put_ep(&ep->parent_ep->com); 995 ep->parent_ep = NULL; 996 } 997 998 static void established_upcall(struct c4iw_ep *ep) 999 { 1000 struct iw_cm_event event; 1001 1002 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1003 memset(&event, 0, sizeof(event)); 1004 event.event = IW_CM_EVENT_ESTABLISHED; 1005 event.ird = ep->ird; 1006 event.ord = ep->ord; 1007 if (ep->com.cm_id) { 1008 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1009 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1010 set_bit(ESTAB_UPCALL, &ep->com.history); 1011 } 1012 } 1013 1014 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 1015 { 1016 struct cpl_rx_data_ack *req; 1017 struct sk_buff *skb; 1018 int wrlen = roundup(sizeof *req, 16); 1019 1020 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 1021 skb = get_skb(NULL, wrlen, GFP_KERNEL); 1022 if (!skb) { 1023 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 1024 return 0; 1025 } 1026 1027 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 1028 memset(req, 0, wrlen); 1029 INIT_TP_WR(req, ep->hwtid); 1030 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1031 ep->hwtid)); 1032 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 1033 F_RX_DACK_CHANGE | 1034 V_RX_DACK_MODE(dack_mode)); 1035 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 1036 c4iw_ofld_send(&ep->com.dev->rdev, skb); 1037 return credits; 1038 } 1039 1040 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 1041 { 1042 struct mpa_message *mpa; 1043 struct mpa_v2_conn_params *mpa_v2_params; 1044 u16 plen; 1045 u16 resp_ird, resp_ord; 1046 u8 rtr_mismatch = 0, insuff_ird = 0; 1047 struct c4iw_qp_attributes attrs; 1048 enum c4iw_qp_attr_mask mask; 1049 int err; 1050 1051 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1052 1053 /* 1054 * Stop mpa timer. If it expired, then the state has 1055 * changed and we bail since ep_timeout already aborted 1056 * the connection. 1057 */ 1058 stop_ep_timer(ep); 1059 if (state_read(&ep->com) != MPA_REQ_SENT) 1060 return; 1061 1062 /* 1063 * If we get more than the supported amount of private data 1064 * then we must fail this connection. 1065 */ 1066 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1067 err = -EINVAL; 1068 goto err; 1069 } 1070 1071 /* 1072 * copy the new data into our accumulation buffer. 1073 */ 1074 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1075 skb->len); 1076 ep->mpa_pkt_len += skb->len; 1077 1078 /* 1079 * if we don't even have the mpa message, then bail. 1080 */ 1081 if (ep->mpa_pkt_len < sizeof(*mpa)) 1082 return; 1083 mpa = (struct mpa_message *) ep->mpa_pkt; 1084 1085 /* Validate MPA header. */ 1086 if (mpa->revision > mpa_rev) { 1087 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1088 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1089 err = -EPROTO; 1090 goto err; 1091 } 1092 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1093 err = -EPROTO; 1094 goto err; 1095 } 1096 1097 plen = ntohs(mpa->private_data_size); 1098 1099 /* 1100 * Fail if there's too much private data. 1101 */ 1102 if (plen > MPA_MAX_PRIVATE_DATA) { 1103 err = -EPROTO; 1104 goto err; 1105 } 1106 1107 /* 1108 * If plen does not account for pkt size 1109 */ 1110 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1111 err = -EPROTO; 1112 goto err; 1113 } 1114 1115 ep->plen = (u8) plen; 1116 1117 /* 1118 * If we don't have all the pdata yet, then bail. 1119 * We'll continue process when more data arrives. 1120 */ 1121 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1122 return; 1123 1124 if (mpa->flags & MPA_REJECT) { 1125 err = -ECONNREFUSED; 1126 goto err; 1127 } 1128 1129 /* 1130 * If we get here we have accumulated the entire mpa 1131 * start reply message including private data. And 1132 * the MPA header is valid. 1133 */ 1134 state_set(&ep->com, FPDU_MODE); 1135 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1136 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1137 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1138 ep->mpa_attr.version = mpa->revision; 1139 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1140 1141 if (mpa->revision == 2) { 1142 ep->mpa_attr.enhanced_rdma_conn = 1143 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1144 if (ep->mpa_attr.enhanced_rdma_conn) { 1145 mpa_v2_params = (struct mpa_v2_conn_params *) 1146 (ep->mpa_pkt + sizeof(*mpa)); 1147 resp_ird = ntohs(mpa_v2_params->ird) & 1148 MPA_V2_IRD_ORD_MASK; 1149 resp_ord = ntohs(mpa_v2_params->ord) & 1150 MPA_V2_IRD_ORD_MASK; 1151 1152 /* 1153 * This is a double-check. Ideally, below checks are 1154 * not required since ird/ord stuff has been taken 1155 * care of in c4iw_accept_cr 1156 */ 1157 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1158 err = -ENOMEM; 1159 ep->ird = resp_ord; 1160 ep->ord = resp_ird; 1161 insuff_ird = 1; 1162 } 1163 1164 if (ntohs(mpa_v2_params->ird) & 1165 MPA_V2_PEER2PEER_MODEL) { 1166 if (ntohs(mpa_v2_params->ord) & 1167 MPA_V2_RDMA_WRITE_RTR) 1168 ep->mpa_attr.p2p_type = 1169 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1170 else if (ntohs(mpa_v2_params->ord) & 1171 MPA_V2_RDMA_READ_RTR) 1172 ep->mpa_attr.p2p_type = 1173 FW_RI_INIT_P2PTYPE_READ_REQ; 1174 } 1175 } 1176 } else if (mpa->revision == 1) 1177 if (peer2peer) 1178 ep->mpa_attr.p2p_type = p2p_type; 1179 1180 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1181 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1182 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1183 ep->mpa_attr.recv_marker_enabled, 1184 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1185 ep->mpa_attr.p2p_type, p2p_type); 1186 1187 /* 1188 * If responder's RTR does not match with that of initiator, assign 1189 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1190 * generated when moving QP to RTS state. 1191 * A TERM message will be sent after QP has moved to RTS state 1192 */ 1193 if ((ep->mpa_attr.version == 2) && peer2peer && 1194 (ep->mpa_attr.p2p_type != p2p_type)) { 1195 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1196 rtr_mismatch = 1; 1197 } 1198 1199 attrs.mpa_attr = ep->mpa_attr; 1200 attrs.max_ird = ep->ird; 1201 attrs.max_ord = ep->ord; 1202 attrs.llp_stream_handle = ep; 1203 attrs.next_state = C4IW_QP_STATE_RTS; 1204 1205 mask = C4IW_QP_ATTR_NEXT_STATE | 1206 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1207 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1208 1209 /* bind QP and TID with INIT_WR */ 1210 err = c4iw_modify_qp(ep->com.qp->rhp, 1211 ep->com.qp, mask, &attrs, 1); 1212 if (err) 1213 goto err; 1214 1215 /* 1216 * If responder's RTR requirement did not match with what initiator 1217 * supports, generate TERM message 1218 */ 1219 if (rtr_mismatch) { 1220 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1221 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1222 attrs.ecode = MPA_NOMATCH_RTR; 1223 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1224 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1225 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1226 err = -ENOMEM; 1227 goto out; 1228 } 1229 1230 /* 1231 * Generate TERM if initiator IRD is not sufficient for responder 1232 * provided ORD. Currently, we do the same behaviour even when 1233 * responder provided IRD is also not sufficient as regards to 1234 * initiator ORD. 1235 */ 1236 if (insuff_ird) { 1237 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1238 __func__); 1239 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1240 attrs.ecode = MPA_INSUFF_IRD; 1241 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1242 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1243 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1244 err = -ENOMEM; 1245 goto out; 1246 } 1247 goto out; 1248 err: 1249 state_set(&ep->com, ABORTING); 1250 send_abort(ep, skb, GFP_KERNEL); 1251 out: 1252 connect_reply_upcall(ep, err); 1253 return; 1254 } 1255 1256 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1257 { 1258 struct mpa_message *mpa; 1259 struct mpa_v2_conn_params *mpa_v2_params; 1260 u16 plen; 1261 1262 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1263 1264 if (state_read(&ep->com) != MPA_REQ_WAIT) 1265 return; 1266 1267 /* 1268 * If we get more than the supported amount of private data 1269 * then we must fail this connection. 1270 */ 1271 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1272 stop_ep_timer(ep); 1273 abort_connection(ep, skb, GFP_KERNEL); 1274 return; 1275 } 1276 1277 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1278 1279 /* 1280 * Copy the new data into our accumulation buffer. 1281 */ 1282 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1283 skb->len); 1284 ep->mpa_pkt_len += skb->len; 1285 1286 /* 1287 * If we don't even have the mpa message, then bail. 1288 * We'll continue process when more data arrives. 1289 */ 1290 if (ep->mpa_pkt_len < sizeof(*mpa)) 1291 return; 1292 1293 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1294 stop_ep_timer(ep); 1295 mpa = (struct mpa_message *) ep->mpa_pkt; 1296 1297 /* 1298 * Validate MPA Header. 1299 */ 1300 if (mpa->revision > mpa_rev) { 1301 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1302 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1303 stop_ep_timer(ep); 1304 abort_connection(ep, skb, GFP_KERNEL); 1305 return; 1306 } 1307 1308 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1309 stop_ep_timer(ep); 1310 abort_connection(ep, skb, GFP_KERNEL); 1311 return; 1312 } 1313 1314 plen = ntohs(mpa->private_data_size); 1315 1316 /* 1317 * Fail if there's too much private data. 1318 */ 1319 if (plen > MPA_MAX_PRIVATE_DATA) { 1320 stop_ep_timer(ep); 1321 abort_connection(ep, skb, GFP_KERNEL); 1322 return; 1323 } 1324 1325 /* 1326 * If plen does not account for pkt size 1327 */ 1328 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1329 stop_ep_timer(ep); 1330 abort_connection(ep, skb, GFP_KERNEL); 1331 return; 1332 } 1333 ep->plen = (u8) plen; 1334 1335 /* 1336 * If we don't have all the pdata yet, then bail. 1337 */ 1338 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1339 return; 1340 1341 /* 1342 * If we get here we have accumulated the entire mpa 1343 * start reply message including private data. 1344 */ 1345 ep->mpa_attr.initiator = 0; 1346 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1347 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1348 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1349 ep->mpa_attr.version = mpa->revision; 1350 if (mpa->revision == 1) 1351 ep->tried_with_mpa_v1 = 1; 1352 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1353 1354 if (mpa->revision == 2) { 1355 ep->mpa_attr.enhanced_rdma_conn = 1356 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1357 if (ep->mpa_attr.enhanced_rdma_conn) { 1358 mpa_v2_params = (struct mpa_v2_conn_params *) 1359 (ep->mpa_pkt + sizeof(*mpa)); 1360 ep->ird = ntohs(mpa_v2_params->ird) & 1361 MPA_V2_IRD_ORD_MASK; 1362 ep->ord = ntohs(mpa_v2_params->ord) & 1363 MPA_V2_IRD_ORD_MASK; 1364 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1365 if (peer2peer) { 1366 if (ntohs(mpa_v2_params->ord) & 1367 MPA_V2_RDMA_WRITE_RTR) 1368 ep->mpa_attr.p2p_type = 1369 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1370 else if (ntohs(mpa_v2_params->ord) & 1371 MPA_V2_RDMA_READ_RTR) 1372 ep->mpa_attr.p2p_type = 1373 FW_RI_INIT_P2PTYPE_READ_REQ; 1374 } 1375 } 1376 } else if (mpa->revision == 1) 1377 if (peer2peer) 1378 ep->mpa_attr.p2p_type = p2p_type; 1379 1380 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1381 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1382 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1383 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1384 ep->mpa_attr.p2p_type); 1385 1386 state_set(&ep->com, MPA_REQ_RCVD); 1387 1388 /* drive upcall */ 1389 connect_request_upcall(ep); 1390 return; 1391 } 1392 1393 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1394 { 1395 struct c4iw_ep *ep; 1396 struct cpl_rx_data *hdr = cplhdr(skb); 1397 unsigned int dlen = ntohs(hdr->len); 1398 unsigned int tid = GET_TID(hdr); 1399 struct tid_info *t = dev->rdev.lldi.tids; 1400 __u8 status = hdr->status; 1401 1402 ep = lookup_tid(t, tid); 1403 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1404 skb_pull(skb, sizeof(*hdr)); 1405 skb_trim(skb, dlen); 1406 1407 /* update RX credits */ 1408 update_rx_credits(ep, dlen); 1409 1410 switch (state_read(&ep->com)) { 1411 case MPA_REQ_SENT: 1412 ep->rcv_seq += dlen; 1413 process_mpa_reply(ep, skb); 1414 break; 1415 case MPA_REQ_WAIT: 1416 ep->rcv_seq += dlen; 1417 process_mpa_request(ep, skb); 1418 break; 1419 case FPDU_MODE: { 1420 struct c4iw_qp_attributes attrs; 1421 BUG_ON(!ep->com.qp); 1422 if (status) 1423 pr_err("%s Unexpected streaming data." \ 1424 " qpid %u ep %p state %d tid %u status %d\n", 1425 __func__, ep->com.qp->wq.sq.qid, ep, 1426 state_read(&ep->com), ep->hwtid, status); 1427 attrs.next_state = C4IW_QP_STATE_ERROR; 1428 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1429 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1430 c4iw_ep_disconnect(ep, 1, GFP_KERNEL); 1431 break; 1432 } 1433 default: 1434 break; 1435 } 1436 return 0; 1437 } 1438 1439 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1440 { 1441 struct c4iw_ep *ep; 1442 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1443 int release = 0; 1444 unsigned int tid = GET_TID(rpl); 1445 struct tid_info *t = dev->rdev.lldi.tids; 1446 1447 ep = lookup_tid(t, tid); 1448 if (!ep) { 1449 printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n"); 1450 return 0; 1451 } 1452 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1453 mutex_lock(&ep->com.mutex); 1454 switch (ep->com.state) { 1455 case ABORTING: 1456 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1457 __state_set(&ep->com, DEAD); 1458 release = 1; 1459 break; 1460 default: 1461 printk(KERN_ERR "%s ep %p state %d\n", 1462 __func__, ep, ep->com.state); 1463 break; 1464 } 1465 mutex_unlock(&ep->com.mutex); 1466 1467 if (release) 1468 release_ep_resources(ep); 1469 return 0; 1470 } 1471 1472 static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid) 1473 { 1474 struct sk_buff *skb; 1475 struct fw_ofld_connection_wr *req; 1476 unsigned int mtu_idx; 1477 int wscale; 1478 1479 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1480 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1481 memset(req, 0, sizeof(*req)); 1482 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR)); 1483 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1484 req->le.filter = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, 1485 ep->l2t)); 1486 req->le.lport = ep->com.local_addr.sin_port; 1487 req->le.pport = ep->com.remote_addr.sin_port; 1488 req->le.u.ipv4.lip = ep->com.local_addr.sin_addr.s_addr; 1489 req->le.u.ipv4.pip = ep->com.remote_addr.sin_addr.s_addr; 1490 req->tcb.t_state_to_astid = 1491 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1492 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1493 req->tcb.cplrxdataack_cplpassacceptrpl = 1494 htons(F_FW_OFLD_CONNECTION_WR_CPLRXDATAACK); 1495 req->tcb.tx_max = (__force __be32) jiffies; 1496 req->tcb.rcv_adv = htons(1); 1497 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1498 wscale = compute_wscale(rcv_win); 1499 req->tcb.opt0 = (__force __be64) (TCAM_BYPASS(1) | 1500 (nocong ? NO_CONG(1) : 0) | 1501 KEEP_ALIVE(1) | 1502 DELACK(1) | 1503 WND_SCALE(wscale) | 1504 MSS_IDX(mtu_idx) | 1505 L2T_IDX(ep->l2t->idx) | 1506 TX_CHAN(ep->tx_chan) | 1507 SMAC_SEL(ep->smac_idx) | 1508 DSCP(ep->tos) | 1509 ULP_MODE(ULP_MODE_TCPDDP) | 1510 RCV_BUFSIZ(rcv_win >> 10)); 1511 req->tcb.opt2 = (__force __be32) (PACE(1) | 1512 TX_QUEUE(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) | 1513 RX_CHANNEL(0) | 1514 CCTRL_ECN(enable_ecn) | 1515 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid)); 1516 if (enable_tcp_timestamps) 1517 req->tcb.opt2 |= (__force __be32) TSTAMPS_EN(1); 1518 if (enable_tcp_sack) 1519 req->tcb.opt2 |= (__force __be32) SACK_EN(1); 1520 if (wscale && enable_tcp_window_scaling) 1521 req->tcb.opt2 |= (__force __be32) WND_SCALE_EN(1); 1522 req->tcb.opt0 = cpu_to_be64((__force u64) req->tcb.opt0); 1523 req->tcb.opt2 = cpu_to_be32((__force u32) req->tcb.opt2); 1524 set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx); 1525 set_bit(ACT_OFLD_CONN, &ep->com.history); 1526 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1527 } 1528 1529 /* 1530 * Return whether a failed active open has allocated a TID 1531 */ 1532 static inline int act_open_has_tid(int status) 1533 { 1534 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1535 status != CPL_ERR_ARP_MISS; 1536 } 1537 1538 #define ACT_OPEN_RETRY_COUNT 2 1539 1540 static int c4iw_reconnect(struct c4iw_ep *ep) 1541 { 1542 int err = 0; 1543 struct rtable *rt; 1544 struct port_info *pi; 1545 struct net_device *pdev; 1546 int step; 1547 struct neighbour *neigh; 1548 1549 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1550 init_timer(&ep->timer); 1551 1552 /* 1553 * Allocate an active TID to initiate a TCP connection. 1554 */ 1555 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1556 if (ep->atid == -1) { 1557 pr_err("%s - cannot alloc atid.\n", __func__); 1558 err = -ENOMEM; 1559 goto fail2; 1560 } 1561 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1562 1563 /* find a route */ 1564 rt = find_route(ep->com.dev, 1565 ep->com.cm_id->local_addr.sin_addr.s_addr, 1566 ep->com.cm_id->remote_addr.sin_addr.s_addr, 1567 ep->com.cm_id->local_addr.sin_port, 1568 ep->com.cm_id->remote_addr.sin_port, 0); 1569 if (!rt) { 1570 pr_err("%s - cannot find route.\n", __func__); 1571 err = -EHOSTUNREACH; 1572 goto fail3; 1573 } 1574 ep->dst = &rt->dst; 1575 1576 neigh = dst_neigh_lookup(ep->dst, 1577 &ep->com.cm_id->remote_addr.sin_addr.s_addr); 1578 /* get a l2t entry */ 1579 if (neigh->dev->flags & IFF_LOOPBACK) { 1580 PDBG("%s LOOPBACK\n", __func__); 1581 pdev = ip_dev_find(&init_net, 1582 ep->com.cm_id->remote_addr.sin_addr.s_addr); 1583 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t, 1584 neigh, pdev, 0); 1585 pi = (struct port_info *)netdev_priv(pdev); 1586 ep->mtu = pdev->mtu; 1587 ep->tx_chan = cxgb4_port_chan(pdev); 1588 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1589 dev_put(pdev); 1590 } else { 1591 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t, 1592 neigh, neigh->dev, 0); 1593 pi = (struct port_info *)netdev_priv(neigh->dev); 1594 ep->mtu = dst_mtu(ep->dst); 1595 ep->tx_chan = cxgb4_port_chan(neigh->dev); 1596 ep->smac_idx = (cxgb4_port_viid(neigh->dev) & 1597 0x7F) << 1; 1598 } 1599 1600 step = ep->com.dev->rdev.lldi.ntxq / ep->com.dev->rdev.lldi.nchan; 1601 ep->txq_idx = pi->port_id * step; 1602 ep->ctrlq_idx = pi->port_id; 1603 step = ep->com.dev->rdev.lldi.nrxq / ep->com.dev->rdev.lldi.nchan; 1604 ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[pi->port_id * step]; 1605 1606 if (!ep->l2t) { 1607 pr_err("%s - cannot alloc l2e.\n", __func__); 1608 err = -ENOMEM; 1609 goto fail4; 1610 } 1611 1612 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1613 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1614 ep->l2t->idx); 1615 1616 state_set(&ep->com, CONNECTING); 1617 ep->tos = 0; 1618 1619 /* send connect request to rnic */ 1620 err = send_connect(ep); 1621 if (!err) 1622 goto out; 1623 1624 cxgb4_l2t_release(ep->l2t); 1625 fail4: 1626 dst_release(ep->dst); 1627 fail3: 1628 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 1629 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1630 fail2: 1631 /* 1632 * remember to send notification to upper layer. 1633 * We are in here so the upper layer is not aware that this is 1634 * re-connect attempt and so, upper layer is still waiting for 1635 * response of 1st connect request. 1636 */ 1637 connect_reply_upcall(ep, -ECONNRESET); 1638 c4iw_put_ep(&ep->com); 1639 out: 1640 return err; 1641 } 1642 1643 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1644 { 1645 struct c4iw_ep *ep; 1646 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1647 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1648 ntohl(rpl->atid_status))); 1649 struct tid_info *t = dev->rdev.lldi.tids; 1650 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1651 1652 ep = lookup_atid(t, atid); 1653 1654 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1655 status, status2errno(status)); 1656 1657 if (status == CPL_ERR_RTX_NEG_ADVICE) { 1658 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1659 atid); 1660 return 0; 1661 } 1662 1663 set_bit(ACT_OPEN_RPL, &ep->com.history); 1664 1665 /* 1666 * Log interesting failures. 1667 */ 1668 switch (status) { 1669 case CPL_ERR_CONN_RESET: 1670 case CPL_ERR_CONN_TIMEDOUT: 1671 break; 1672 case CPL_ERR_TCAM_FULL: 1673 if (dev->rdev.lldi.enable_fw_ofld_conn) { 1674 mutex_lock(&dev->rdev.stats.lock); 1675 dev->rdev.stats.tcam_full++; 1676 mutex_unlock(&dev->rdev.stats.lock); 1677 send_fw_act_open_req(ep, 1678 GET_TID_TID(GET_AOPEN_ATID( 1679 ntohl(rpl->atid_status)))); 1680 return 0; 1681 } 1682 break; 1683 case CPL_ERR_CONN_EXIST: 1684 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 1685 set_bit(ACT_RETRY_INUSE, &ep->com.history); 1686 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, 1687 atid); 1688 cxgb4_free_atid(t, atid); 1689 dst_release(ep->dst); 1690 cxgb4_l2t_release(ep->l2t); 1691 c4iw_reconnect(ep); 1692 return 0; 1693 } 1694 break; 1695 default: 1696 printk(KERN_INFO MOD "Active open failure - " 1697 "atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1698 atid, status, status2errno(status), 1699 &ep->com.local_addr.sin_addr.s_addr, 1700 ntohs(ep->com.local_addr.sin_port), 1701 &ep->com.remote_addr.sin_addr.s_addr, 1702 ntohs(ep->com.remote_addr.sin_port)); 1703 break; 1704 } 1705 1706 connect_reply_upcall(ep, status2errno(status)); 1707 state_set(&ep->com, DEAD); 1708 1709 if (status && act_open_has_tid(status)) 1710 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1711 1712 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 1713 cxgb4_free_atid(t, atid); 1714 dst_release(ep->dst); 1715 cxgb4_l2t_release(ep->l2t); 1716 c4iw_put_ep(&ep->com); 1717 1718 return 0; 1719 } 1720 1721 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1722 { 1723 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1724 struct tid_info *t = dev->rdev.lldi.tids; 1725 unsigned int stid = GET_TID(rpl); 1726 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1727 1728 if (!ep) { 1729 PDBG("%s stid %d lookup failure!\n", __func__, stid); 1730 goto out; 1731 } 1732 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1733 rpl->status, status2errno(rpl->status)); 1734 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1735 1736 out: 1737 return 0; 1738 } 1739 1740 static int listen_stop(struct c4iw_listen_ep *ep) 1741 { 1742 struct sk_buff *skb; 1743 struct cpl_close_listsvr_req *req; 1744 1745 PDBG("%s ep %p\n", __func__, ep); 1746 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1747 if (!skb) { 1748 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 1749 return -ENOMEM; 1750 } 1751 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req)); 1752 INIT_TP_WR(req, 0); 1753 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 1754 ep->stid)); 1755 req->reply_ctrl = cpu_to_be16( 1756 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0])); 1757 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 1758 return c4iw_ofld_send(&ep->com.dev->rdev, skb); 1759 } 1760 1761 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1762 { 1763 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1764 struct tid_info *t = dev->rdev.lldi.tids; 1765 unsigned int stid = GET_TID(rpl); 1766 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1767 1768 PDBG("%s ep %p\n", __func__, ep); 1769 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1770 return 0; 1771 } 1772 1773 static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, 1774 struct cpl_pass_accept_req *req) 1775 { 1776 struct cpl_pass_accept_rpl *rpl; 1777 unsigned int mtu_idx; 1778 u64 opt0; 1779 u32 opt2; 1780 int wscale; 1781 1782 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1783 BUG_ON(skb_cloned(skb)); 1784 skb_trim(skb, sizeof(*rpl)); 1785 skb_get(skb); 1786 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1787 wscale = compute_wscale(rcv_win); 1788 opt0 = (nocong ? NO_CONG(1) : 0) | 1789 KEEP_ALIVE(1) | 1790 DELACK(1) | 1791 WND_SCALE(wscale) | 1792 MSS_IDX(mtu_idx) | 1793 L2T_IDX(ep->l2t->idx) | 1794 TX_CHAN(ep->tx_chan) | 1795 SMAC_SEL(ep->smac_idx) | 1796 DSCP(ep->tos >> 2) | 1797 ULP_MODE(ULP_MODE_TCPDDP) | 1798 RCV_BUFSIZ(rcv_win>>10); 1799 opt2 = RX_CHANNEL(0) | 1800 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1801 1802 if (enable_tcp_timestamps && req->tcpopt.tstamp) 1803 opt2 |= TSTAMPS_EN(1); 1804 if (enable_tcp_sack && req->tcpopt.sack) 1805 opt2 |= SACK_EN(1); 1806 if (wscale && enable_tcp_window_scaling) 1807 opt2 |= WND_SCALE_EN(1); 1808 if (enable_ecn) { 1809 const struct tcphdr *tcph; 1810 u32 hlen = ntohl(req->hdr_len); 1811 1812 tcph = (const void *)(req + 1) + G_ETH_HDR_LEN(hlen) + 1813 G_IP_HDR_LEN(hlen); 1814 if (tcph->ece && tcph->cwr) 1815 opt2 |= CCTRL_ECN(1); 1816 } 1817 1818 rpl = cplhdr(skb); 1819 INIT_TP_WR(rpl, ep->hwtid); 1820 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1821 ep->hwtid)); 1822 rpl->opt0 = cpu_to_be64(opt0); 1823 rpl->opt2 = cpu_to_be32(opt2); 1824 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 1825 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1826 1827 return; 1828 } 1829 1830 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip, 1831 struct sk_buff *skb) 1832 { 1833 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid, 1834 peer_ip); 1835 BUG_ON(skb_cloned(skb)); 1836 skb_trim(skb, sizeof(struct cpl_tid_release)); 1837 skb_get(skb); 1838 release_tid(&dev->rdev, hwtid, skb); 1839 return; 1840 } 1841 1842 static void get_4tuple(struct cpl_pass_accept_req *req, 1843 __be32 *local_ip, __be32 *peer_ip, 1844 __be16 *local_port, __be16 *peer_port) 1845 { 1846 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 1847 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 1848 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 1849 struct tcphdr *tcp = (struct tcphdr *) 1850 ((u8 *)(req + 1) + eth_len + ip_len); 1851 1852 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 1853 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 1854 ntohs(tcp->dest)); 1855 1856 *peer_ip = ip->saddr; 1857 *local_ip = ip->daddr; 1858 *peer_port = tcp->source; 1859 *local_port = tcp->dest; 1860 1861 return; 1862 } 1863 1864 static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst, 1865 struct c4iw_dev *cdev, bool clear_mpa_v1) 1866 { 1867 struct neighbour *n; 1868 int err, step; 1869 1870 n = dst_neigh_lookup(dst, &peer_ip); 1871 if (!n) 1872 return -ENODEV; 1873 1874 rcu_read_lock(); 1875 err = -ENOMEM; 1876 if (n->dev->flags & IFF_LOOPBACK) { 1877 struct net_device *pdev; 1878 1879 pdev = ip_dev_find(&init_net, peer_ip); 1880 if (!pdev) { 1881 err = -ENODEV; 1882 goto out; 1883 } 1884 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1885 n, pdev, 0); 1886 if (!ep->l2t) 1887 goto out; 1888 ep->mtu = pdev->mtu; 1889 ep->tx_chan = cxgb4_port_chan(pdev); 1890 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1891 step = cdev->rdev.lldi.ntxq / 1892 cdev->rdev.lldi.nchan; 1893 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1894 step = cdev->rdev.lldi.nrxq / 1895 cdev->rdev.lldi.nchan; 1896 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1897 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1898 cxgb4_port_idx(pdev) * step]; 1899 dev_put(pdev); 1900 } else { 1901 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1902 n, n->dev, 0); 1903 if (!ep->l2t) 1904 goto out; 1905 ep->mtu = dst_mtu(dst); 1906 ep->tx_chan = cxgb4_port_chan(n->dev); 1907 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1908 step = cdev->rdev.lldi.ntxq / 1909 cdev->rdev.lldi.nchan; 1910 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1911 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1912 step = cdev->rdev.lldi.nrxq / 1913 cdev->rdev.lldi.nchan; 1914 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1915 cxgb4_port_idx(n->dev) * step]; 1916 1917 if (clear_mpa_v1) { 1918 ep->retry_with_mpa_v1 = 0; 1919 ep->tried_with_mpa_v1 = 0; 1920 } 1921 } 1922 err = 0; 1923 out: 1924 rcu_read_unlock(); 1925 1926 neigh_release(n); 1927 1928 return err; 1929 } 1930 1931 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 1932 { 1933 struct c4iw_ep *child_ep = NULL, *parent_ep; 1934 struct cpl_pass_accept_req *req = cplhdr(skb); 1935 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 1936 struct tid_info *t = dev->rdev.lldi.tids; 1937 unsigned int hwtid = GET_TID(req); 1938 struct dst_entry *dst; 1939 struct rtable *rt; 1940 __be32 local_ip, peer_ip = 0; 1941 __be16 local_port, peer_port; 1942 int err; 1943 u16 peer_mss = ntohs(req->tcpopt.mss); 1944 1945 parent_ep = lookup_stid(t, stid); 1946 if (!parent_ep) { 1947 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 1948 goto reject; 1949 } 1950 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port); 1951 1952 PDBG("%s parent ep %p hwtid %u laddr 0x%x raddr 0x%x lport %d " \ 1953 "rport %d peer_mss %d\n", __func__, parent_ep, hwtid, 1954 ntohl(local_ip), ntohl(peer_ip), ntohs(local_port), 1955 ntohs(peer_port), peer_mss); 1956 1957 if (state_read(&parent_ep->com) != LISTEN) { 1958 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 1959 __func__); 1960 goto reject; 1961 } 1962 1963 /* Find output route */ 1964 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port, 1965 GET_POPEN_TOS(ntohl(req->tos_stid))); 1966 if (!rt) { 1967 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 1968 __func__); 1969 goto reject; 1970 } 1971 dst = &rt->dst; 1972 1973 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 1974 if (!child_ep) { 1975 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 1976 __func__); 1977 dst_release(dst); 1978 goto reject; 1979 } 1980 1981 err = import_ep(child_ep, peer_ip, dst, dev, false); 1982 if (err) { 1983 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 1984 __func__); 1985 dst_release(dst); 1986 kfree(child_ep); 1987 goto reject; 1988 } 1989 1990 if (peer_mss && child_ep->mtu > (peer_mss + 40)) 1991 child_ep->mtu = peer_mss + 40; 1992 1993 state_set(&child_ep->com, CONNECTING); 1994 child_ep->com.dev = dev; 1995 child_ep->com.cm_id = NULL; 1996 child_ep->com.local_addr.sin_family = PF_INET; 1997 child_ep->com.local_addr.sin_port = local_port; 1998 child_ep->com.local_addr.sin_addr.s_addr = local_ip; 1999 child_ep->com.remote_addr.sin_family = PF_INET; 2000 child_ep->com.remote_addr.sin_port = peer_port; 2001 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip; 2002 c4iw_get_ep(&parent_ep->com); 2003 child_ep->parent_ep = parent_ep; 2004 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 2005 child_ep->dst = dst; 2006 child_ep->hwtid = hwtid; 2007 2008 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 2009 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 2010 2011 init_timer(&child_ep->timer); 2012 cxgb4_insert_tid(t, child_ep, hwtid); 2013 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid); 2014 accept_cr(child_ep, peer_ip, skb, req); 2015 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 2016 goto out; 2017 reject: 2018 reject_cr(dev, hwtid, peer_ip, skb); 2019 out: 2020 return 0; 2021 } 2022 2023 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 2024 { 2025 struct c4iw_ep *ep; 2026 struct cpl_pass_establish *req = cplhdr(skb); 2027 struct tid_info *t = dev->rdev.lldi.tids; 2028 unsigned int tid = GET_TID(req); 2029 2030 ep = lookup_tid(t, tid); 2031 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2032 ep->snd_seq = be32_to_cpu(req->snd_isn); 2033 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 2034 2035 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__, ep, tid, 2036 ntohs(req->tcp_opt)); 2037 2038 set_emss(ep, ntohs(req->tcp_opt)); 2039 2040 dst_confirm(ep->dst); 2041 state_set(&ep->com, MPA_REQ_WAIT); 2042 start_ep_timer(ep); 2043 send_flowc(ep, skb); 2044 set_bit(PASS_ESTAB, &ep->com.history); 2045 2046 return 0; 2047 } 2048 2049 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 2050 { 2051 struct cpl_peer_close *hdr = cplhdr(skb); 2052 struct c4iw_ep *ep; 2053 struct c4iw_qp_attributes attrs; 2054 int disconnect = 1; 2055 int release = 0; 2056 struct tid_info *t = dev->rdev.lldi.tids; 2057 unsigned int tid = GET_TID(hdr); 2058 int ret; 2059 2060 ep = lookup_tid(t, tid); 2061 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2062 dst_confirm(ep->dst); 2063 2064 set_bit(PEER_CLOSE, &ep->com.history); 2065 mutex_lock(&ep->com.mutex); 2066 switch (ep->com.state) { 2067 case MPA_REQ_WAIT: 2068 __state_set(&ep->com, CLOSING); 2069 break; 2070 case MPA_REQ_SENT: 2071 __state_set(&ep->com, CLOSING); 2072 connect_reply_upcall(ep, -ECONNRESET); 2073 break; 2074 case MPA_REQ_RCVD: 2075 2076 /* 2077 * We're gonna mark this puppy DEAD, but keep 2078 * the reference on it until the ULP accepts or 2079 * rejects the CR. Also wake up anyone waiting 2080 * in rdma connection migration (see c4iw_accept_cr()). 2081 */ 2082 __state_set(&ep->com, CLOSING); 2083 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2084 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2085 break; 2086 case MPA_REP_SENT: 2087 __state_set(&ep->com, CLOSING); 2088 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2089 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2090 break; 2091 case FPDU_MODE: 2092 start_ep_timer(ep); 2093 __state_set(&ep->com, CLOSING); 2094 attrs.next_state = C4IW_QP_STATE_CLOSING; 2095 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2096 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2097 if (ret != -ECONNRESET) { 2098 peer_close_upcall(ep); 2099 disconnect = 1; 2100 } 2101 break; 2102 case ABORTING: 2103 disconnect = 0; 2104 break; 2105 case CLOSING: 2106 __state_set(&ep->com, MORIBUND); 2107 disconnect = 0; 2108 break; 2109 case MORIBUND: 2110 stop_ep_timer(ep); 2111 if (ep->com.cm_id && ep->com.qp) { 2112 attrs.next_state = C4IW_QP_STATE_IDLE; 2113 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2114 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2115 } 2116 close_complete_upcall(ep); 2117 __state_set(&ep->com, DEAD); 2118 release = 1; 2119 disconnect = 0; 2120 break; 2121 case DEAD: 2122 disconnect = 0; 2123 break; 2124 default: 2125 BUG_ON(1); 2126 } 2127 mutex_unlock(&ep->com.mutex); 2128 if (disconnect) 2129 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2130 if (release) 2131 release_ep_resources(ep); 2132 return 0; 2133 } 2134 2135 /* 2136 * Returns whether an ABORT_REQ_RSS message is a negative advice. 2137 */ 2138 static int is_neg_adv_abort(unsigned int status) 2139 { 2140 return status == CPL_ERR_RTX_NEG_ADVICE || 2141 status == CPL_ERR_PERSIST_NEG_ADVICE; 2142 } 2143 2144 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 2145 { 2146 struct cpl_abort_req_rss *req = cplhdr(skb); 2147 struct c4iw_ep *ep; 2148 struct cpl_abort_rpl *rpl; 2149 struct sk_buff *rpl_skb; 2150 struct c4iw_qp_attributes attrs; 2151 int ret; 2152 int release = 0; 2153 struct tid_info *t = dev->rdev.lldi.tids; 2154 unsigned int tid = GET_TID(req); 2155 2156 ep = lookup_tid(t, tid); 2157 if (is_neg_adv_abort(req->status)) { 2158 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2159 ep->hwtid); 2160 return 0; 2161 } 2162 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2163 ep->com.state); 2164 set_bit(PEER_ABORT, &ep->com.history); 2165 2166 /* 2167 * Wake up any threads in rdma_init() or rdma_fini(). 2168 * However, this is not needed if com state is just 2169 * MPA_REQ_SENT 2170 */ 2171 if (ep->com.state != MPA_REQ_SENT) 2172 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2173 2174 mutex_lock(&ep->com.mutex); 2175 switch (ep->com.state) { 2176 case CONNECTING: 2177 break; 2178 case MPA_REQ_WAIT: 2179 stop_ep_timer(ep); 2180 break; 2181 case MPA_REQ_SENT: 2182 stop_ep_timer(ep); 2183 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1)) 2184 connect_reply_upcall(ep, -ECONNRESET); 2185 else { 2186 /* 2187 * we just don't send notification upwards because we 2188 * want to retry with mpa_v1 without upper layers even 2189 * knowing it. 2190 * 2191 * do some housekeeping so as to re-initiate the 2192 * connection 2193 */ 2194 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 2195 mpa_rev); 2196 ep->retry_with_mpa_v1 = 1; 2197 } 2198 break; 2199 case MPA_REP_SENT: 2200 break; 2201 case MPA_REQ_RCVD: 2202 break; 2203 case MORIBUND: 2204 case CLOSING: 2205 stop_ep_timer(ep); 2206 /*FALLTHROUGH*/ 2207 case FPDU_MODE: 2208 if (ep->com.cm_id && ep->com.qp) { 2209 attrs.next_state = C4IW_QP_STATE_ERROR; 2210 ret = c4iw_modify_qp(ep->com.qp->rhp, 2211 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2212 &attrs, 1); 2213 if (ret) 2214 printk(KERN_ERR MOD 2215 "%s - qp <- error failed!\n", 2216 __func__); 2217 } 2218 peer_abort_upcall(ep); 2219 break; 2220 case ABORTING: 2221 break; 2222 case DEAD: 2223 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 2224 mutex_unlock(&ep->com.mutex); 2225 return 0; 2226 default: 2227 BUG_ON(1); 2228 break; 2229 } 2230 dst_confirm(ep->dst); 2231 if (ep->com.state != ABORTING) { 2232 __state_set(&ep->com, DEAD); 2233 /* we don't release if we want to retry with mpa_v1 */ 2234 if (!ep->retry_with_mpa_v1) 2235 release = 1; 2236 } 2237 mutex_unlock(&ep->com.mutex); 2238 2239 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 2240 if (!rpl_skb) { 2241 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 2242 __func__); 2243 release = 1; 2244 goto out; 2245 } 2246 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 2247 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 2248 INIT_TP_WR(rpl, ep->hwtid); 2249 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 2250 rpl->cmd = CPL_ABORT_NO_RST; 2251 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 2252 out: 2253 if (release) 2254 release_ep_resources(ep); 2255 else if (ep->retry_with_mpa_v1) { 2256 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 2257 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 2258 dst_release(ep->dst); 2259 cxgb4_l2t_release(ep->l2t); 2260 c4iw_reconnect(ep); 2261 } 2262 2263 return 0; 2264 } 2265 2266 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2267 { 2268 struct c4iw_ep *ep; 2269 struct c4iw_qp_attributes attrs; 2270 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2271 int release = 0; 2272 struct tid_info *t = dev->rdev.lldi.tids; 2273 unsigned int tid = GET_TID(rpl); 2274 2275 ep = lookup_tid(t, tid); 2276 2277 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2278 BUG_ON(!ep); 2279 2280 /* The cm_id may be null if we failed to connect */ 2281 mutex_lock(&ep->com.mutex); 2282 switch (ep->com.state) { 2283 case CLOSING: 2284 __state_set(&ep->com, MORIBUND); 2285 break; 2286 case MORIBUND: 2287 stop_ep_timer(ep); 2288 if ((ep->com.cm_id) && (ep->com.qp)) { 2289 attrs.next_state = C4IW_QP_STATE_IDLE; 2290 c4iw_modify_qp(ep->com.qp->rhp, 2291 ep->com.qp, 2292 C4IW_QP_ATTR_NEXT_STATE, 2293 &attrs, 1); 2294 } 2295 close_complete_upcall(ep); 2296 __state_set(&ep->com, DEAD); 2297 release = 1; 2298 break; 2299 case ABORTING: 2300 case DEAD: 2301 break; 2302 default: 2303 BUG_ON(1); 2304 break; 2305 } 2306 mutex_unlock(&ep->com.mutex); 2307 if (release) 2308 release_ep_resources(ep); 2309 return 0; 2310 } 2311 2312 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2313 { 2314 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2315 struct tid_info *t = dev->rdev.lldi.tids; 2316 unsigned int tid = GET_TID(rpl); 2317 struct c4iw_ep *ep; 2318 struct c4iw_qp_attributes attrs; 2319 2320 ep = lookup_tid(t, tid); 2321 BUG_ON(!ep); 2322 2323 if (ep && ep->com.qp) { 2324 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2325 ep->com.qp->wq.sq.qid); 2326 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2327 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2328 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2329 } else 2330 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2331 2332 return 0; 2333 } 2334 2335 /* 2336 * Upcall from the adapter indicating data has been transmitted. 2337 * For us its just the single MPA request or reply. We can now free 2338 * the skb holding the mpa message. 2339 */ 2340 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2341 { 2342 struct c4iw_ep *ep; 2343 struct cpl_fw4_ack *hdr = cplhdr(skb); 2344 u8 credits = hdr->credits; 2345 unsigned int tid = GET_TID(hdr); 2346 struct tid_info *t = dev->rdev.lldi.tids; 2347 2348 2349 ep = lookup_tid(t, tid); 2350 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2351 if (credits == 0) { 2352 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2353 __func__, ep, ep->hwtid, state_read(&ep->com)); 2354 return 0; 2355 } 2356 2357 dst_confirm(ep->dst); 2358 if (ep->mpa_skb) { 2359 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2360 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2361 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2362 kfree_skb(ep->mpa_skb); 2363 ep->mpa_skb = NULL; 2364 } 2365 return 0; 2366 } 2367 2368 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2369 { 2370 int err; 2371 struct c4iw_ep *ep = to_ep(cm_id); 2372 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2373 2374 if (state_read(&ep->com) == DEAD) { 2375 c4iw_put_ep(&ep->com); 2376 return -ECONNRESET; 2377 } 2378 set_bit(ULP_REJECT, &ep->com.history); 2379 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2380 if (mpa_rev == 0) 2381 abort_connection(ep, NULL, GFP_KERNEL); 2382 else { 2383 err = send_mpa_reject(ep, pdata, pdata_len); 2384 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2385 } 2386 c4iw_put_ep(&ep->com); 2387 return 0; 2388 } 2389 2390 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2391 { 2392 int err; 2393 struct c4iw_qp_attributes attrs; 2394 enum c4iw_qp_attr_mask mask; 2395 struct c4iw_ep *ep = to_ep(cm_id); 2396 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2397 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2398 2399 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2400 if (state_read(&ep->com) == DEAD) { 2401 err = -ECONNRESET; 2402 goto err; 2403 } 2404 2405 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2406 BUG_ON(!qp); 2407 2408 set_bit(ULP_ACCEPT, &ep->com.history); 2409 if ((conn_param->ord > c4iw_max_read_depth) || 2410 (conn_param->ird > c4iw_max_read_depth)) { 2411 abort_connection(ep, NULL, GFP_KERNEL); 2412 err = -EINVAL; 2413 goto err; 2414 } 2415 2416 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2417 if (conn_param->ord > ep->ird) { 2418 ep->ird = conn_param->ird; 2419 ep->ord = conn_param->ord; 2420 send_mpa_reject(ep, conn_param->private_data, 2421 conn_param->private_data_len); 2422 abort_connection(ep, NULL, GFP_KERNEL); 2423 err = -ENOMEM; 2424 goto err; 2425 } 2426 if (conn_param->ird > ep->ord) { 2427 if (!ep->ord) 2428 conn_param->ird = 1; 2429 else { 2430 abort_connection(ep, NULL, GFP_KERNEL); 2431 err = -ENOMEM; 2432 goto err; 2433 } 2434 } 2435 2436 } 2437 ep->ird = conn_param->ird; 2438 ep->ord = conn_param->ord; 2439 2440 if (ep->mpa_attr.version != 2) 2441 if (peer2peer && ep->ird == 0) 2442 ep->ird = 1; 2443 2444 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2445 2446 cm_id->add_ref(cm_id); 2447 ep->com.cm_id = cm_id; 2448 ep->com.qp = qp; 2449 ref_qp(ep); 2450 2451 /* bind QP to EP and move to RTS */ 2452 attrs.mpa_attr = ep->mpa_attr; 2453 attrs.max_ird = ep->ird; 2454 attrs.max_ord = ep->ord; 2455 attrs.llp_stream_handle = ep; 2456 attrs.next_state = C4IW_QP_STATE_RTS; 2457 2458 /* bind QP and TID with INIT_WR */ 2459 mask = C4IW_QP_ATTR_NEXT_STATE | 2460 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2461 C4IW_QP_ATTR_MPA_ATTR | 2462 C4IW_QP_ATTR_MAX_IRD | 2463 C4IW_QP_ATTR_MAX_ORD; 2464 2465 err = c4iw_modify_qp(ep->com.qp->rhp, 2466 ep->com.qp, mask, &attrs, 1); 2467 if (err) 2468 goto err1; 2469 err = send_mpa_reply(ep, conn_param->private_data, 2470 conn_param->private_data_len); 2471 if (err) 2472 goto err1; 2473 2474 state_set(&ep->com, FPDU_MODE); 2475 established_upcall(ep); 2476 c4iw_put_ep(&ep->com); 2477 return 0; 2478 err1: 2479 ep->com.cm_id = NULL; 2480 cm_id->rem_ref(cm_id); 2481 err: 2482 c4iw_put_ep(&ep->com); 2483 return err; 2484 } 2485 2486 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2487 { 2488 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2489 struct c4iw_ep *ep; 2490 struct rtable *rt; 2491 int err = 0; 2492 2493 if ((conn_param->ord > c4iw_max_read_depth) || 2494 (conn_param->ird > c4iw_max_read_depth)) { 2495 err = -EINVAL; 2496 goto out; 2497 } 2498 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2499 if (!ep) { 2500 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2501 err = -ENOMEM; 2502 goto out; 2503 } 2504 init_timer(&ep->timer); 2505 ep->plen = conn_param->private_data_len; 2506 if (ep->plen) 2507 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2508 conn_param->private_data, ep->plen); 2509 ep->ird = conn_param->ird; 2510 ep->ord = conn_param->ord; 2511 2512 if (peer2peer && ep->ord == 0) 2513 ep->ord = 1; 2514 2515 cm_id->add_ref(cm_id); 2516 ep->com.dev = dev; 2517 ep->com.cm_id = cm_id; 2518 ep->com.qp = get_qhp(dev, conn_param->qpn); 2519 BUG_ON(!ep->com.qp); 2520 ref_qp(ep); 2521 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2522 ep->com.qp, cm_id); 2523 2524 /* 2525 * Allocate an active TID to initiate a TCP connection. 2526 */ 2527 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2528 if (ep->atid == -1) { 2529 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2530 err = -ENOMEM; 2531 goto fail2; 2532 } 2533 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2534 2535 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__, 2536 ntohl(cm_id->local_addr.sin_addr.s_addr), 2537 ntohs(cm_id->local_addr.sin_port), 2538 ntohl(cm_id->remote_addr.sin_addr.s_addr), 2539 ntohs(cm_id->remote_addr.sin_port)); 2540 2541 /* find a route */ 2542 rt = find_route(dev, 2543 cm_id->local_addr.sin_addr.s_addr, 2544 cm_id->remote_addr.sin_addr.s_addr, 2545 cm_id->local_addr.sin_port, 2546 cm_id->remote_addr.sin_port, 0); 2547 if (!rt) { 2548 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2549 err = -EHOSTUNREACH; 2550 goto fail3; 2551 } 2552 ep->dst = &rt->dst; 2553 2554 err = import_ep(ep, cm_id->remote_addr.sin_addr.s_addr, 2555 ep->dst, ep->com.dev, true); 2556 if (err) { 2557 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2558 goto fail4; 2559 } 2560 2561 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2562 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2563 ep->l2t->idx); 2564 2565 state_set(&ep->com, CONNECTING); 2566 ep->tos = 0; 2567 ep->com.local_addr = cm_id->local_addr; 2568 ep->com.remote_addr = cm_id->remote_addr; 2569 2570 /* send connect request to rnic */ 2571 err = send_connect(ep); 2572 if (!err) 2573 goto out; 2574 2575 cxgb4_l2t_release(ep->l2t); 2576 fail4: 2577 dst_release(ep->dst); 2578 fail3: 2579 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 2580 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2581 fail2: 2582 cm_id->rem_ref(cm_id); 2583 c4iw_put_ep(&ep->com); 2584 out: 2585 return err; 2586 } 2587 2588 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2589 { 2590 int err = 0; 2591 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2592 struct c4iw_listen_ep *ep; 2593 2594 might_sleep(); 2595 2596 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2597 if (!ep) { 2598 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2599 err = -ENOMEM; 2600 goto fail1; 2601 } 2602 PDBG("%s ep %p\n", __func__, ep); 2603 cm_id->add_ref(cm_id); 2604 ep->com.cm_id = cm_id; 2605 ep->com.dev = dev; 2606 ep->backlog = backlog; 2607 ep->com.local_addr = cm_id->local_addr; 2608 2609 /* 2610 * Allocate a server TID. 2611 */ 2612 if (dev->rdev.lldi.enable_fw_ofld_conn) 2613 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, PF_INET, ep); 2614 else 2615 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep); 2616 2617 if (ep->stid == -1) { 2618 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2619 err = -ENOMEM; 2620 goto fail2; 2621 } 2622 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2623 state_set(&ep->com, LISTEN); 2624 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2625 do { 2626 err = cxgb4_create_server_filter( 2627 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2628 ep->com.local_addr.sin_addr.s_addr, 2629 ep->com.local_addr.sin_port, 2630 0, 2631 ep->com.dev->rdev.lldi.rxq_ids[0], 2632 0, 2633 0); 2634 if (err == -EBUSY) { 2635 set_current_state(TASK_UNINTERRUPTIBLE); 2636 schedule_timeout(usecs_to_jiffies(100)); 2637 } 2638 } while (err == -EBUSY); 2639 } else { 2640 c4iw_init_wr_wait(&ep->com.wr_wait); 2641 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], 2642 ep->stid, ep->com.local_addr.sin_addr.s_addr, 2643 ep->com.local_addr.sin_port, 2644 0, 2645 ep->com.dev->rdev.lldi.rxq_ids[0]); 2646 if (!err) 2647 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2648 &ep->com.wr_wait, 2649 0, 0, __func__); 2650 } 2651 if (!err) { 2652 cm_id->provider_data = ep; 2653 goto out; 2654 } 2655 pr_err("%s cxgb4_create_server/filter failed err %d " \ 2656 "stid %d laddr %08x lport %d\n", \ 2657 __func__, err, ep->stid, 2658 ntohl(ep->com.local_addr.sin_addr.s_addr), 2659 ntohs(ep->com.local_addr.sin_port)); 2660 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2661 fail2: 2662 cm_id->rem_ref(cm_id); 2663 c4iw_put_ep(&ep->com); 2664 fail1: 2665 out: 2666 return err; 2667 } 2668 2669 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2670 { 2671 int err; 2672 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2673 2674 PDBG("%s ep %p\n", __func__, ep); 2675 2676 might_sleep(); 2677 state_set(&ep->com, DEAD); 2678 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn) { 2679 err = cxgb4_remove_server_filter( 2680 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2681 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2682 } else { 2683 c4iw_init_wr_wait(&ep->com.wr_wait); 2684 err = listen_stop(ep); 2685 if (err) 2686 goto done; 2687 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 2688 0, 0, __func__); 2689 } 2690 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 2691 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2692 done: 2693 cm_id->rem_ref(cm_id); 2694 c4iw_put_ep(&ep->com); 2695 return err; 2696 } 2697 2698 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2699 { 2700 int ret = 0; 2701 int close = 0; 2702 int fatal = 0; 2703 struct c4iw_rdev *rdev; 2704 2705 mutex_lock(&ep->com.mutex); 2706 2707 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 2708 states[ep->com.state], abrupt); 2709 2710 rdev = &ep->com.dev->rdev; 2711 if (c4iw_fatal_error(rdev)) { 2712 fatal = 1; 2713 close_complete_upcall(ep); 2714 ep->com.state = DEAD; 2715 } 2716 switch (ep->com.state) { 2717 case MPA_REQ_WAIT: 2718 case MPA_REQ_SENT: 2719 case MPA_REQ_RCVD: 2720 case MPA_REP_SENT: 2721 case FPDU_MODE: 2722 close = 1; 2723 if (abrupt) 2724 ep->com.state = ABORTING; 2725 else { 2726 ep->com.state = CLOSING; 2727 start_ep_timer(ep); 2728 } 2729 set_bit(CLOSE_SENT, &ep->com.flags); 2730 break; 2731 case CLOSING: 2732 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2733 close = 1; 2734 if (abrupt) { 2735 stop_ep_timer(ep); 2736 ep->com.state = ABORTING; 2737 } else 2738 ep->com.state = MORIBUND; 2739 } 2740 break; 2741 case MORIBUND: 2742 case ABORTING: 2743 case DEAD: 2744 PDBG("%s ignoring disconnect ep %p state %u\n", 2745 __func__, ep, ep->com.state); 2746 break; 2747 default: 2748 BUG(); 2749 break; 2750 } 2751 2752 if (close) { 2753 if (abrupt) { 2754 set_bit(EP_DISC_ABORT, &ep->com.history); 2755 close_complete_upcall(ep); 2756 ret = send_abort(ep, NULL, gfp); 2757 } else { 2758 set_bit(EP_DISC_CLOSE, &ep->com.history); 2759 ret = send_halfclose(ep, gfp); 2760 } 2761 if (ret) 2762 fatal = 1; 2763 } 2764 mutex_unlock(&ep->com.mutex); 2765 if (fatal) 2766 release_ep_resources(ep); 2767 return ret; 2768 } 2769 2770 static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 2771 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 2772 { 2773 struct c4iw_ep *ep; 2774 int atid = be32_to_cpu(req->tid); 2775 2776 ep = (struct c4iw_ep *)lookup_atid(dev->rdev.lldi.tids, 2777 (__force u32) req->tid); 2778 if (!ep) 2779 return; 2780 2781 switch (req->retval) { 2782 case FW_ENOMEM: 2783 set_bit(ACT_RETRY_NOMEM, &ep->com.history); 2784 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 2785 send_fw_act_open_req(ep, atid); 2786 return; 2787 } 2788 case FW_EADDRINUSE: 2789 set_bit(ACT_RETRY_INUSE, &ep->com.history); 2790 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 2791 send_fw_act_open_req(ep, atid); 2792 return; 2793 } 2794 break; 2795 default: 2796 pr_info("%s unexpected ofld conn wr retval %d\n", 2797 __func__, req->retval); 2798 break; 2799 } 2800 pr_err("active ofld_connect_wr failure %d atid %d\n", 2801 req->retval, atid); 2802 mutex_lock(&dev->rdev.stats.lock); 2803 dev->rdev.stats.act_ofld_conn_fails++; 2804 mutex_unlock(&dev->rdev.stats.lock); 2805 connect_reply_upcall(ep, status2errno(req->retval)); 2806 state_set(&ep->com, DEAD); 2807 remove_handle(dev, &dev->atid_idr, atid); 2808 cxgb4_free_atid(dev->rdev.lldi.tids, atid); 2809 dst_release(ep->dst); 2810 cxgb4_l2t_release(ep->l2t); 2811 c4iw_put_ep(&ep->com); 2812 } 2813 2814 static void passive_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 2815 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 2816 { 2817 struct sk_buff *rpl_skb; 2818 struct cpl_pass_accept_req *cpl; 2819 int ret; 2820 2821 rpl_skb = (struct sk_buff *)(unsigned long)req->cookie; 2822 BUG_ON(!rpl_skb); 2823 if (req->retval) { 2824 PDBG("%s passive open failure %d\n", __func__, req->retval); 2825 mutex_lock(&dev->rdev.stats.lock); 2826 dev->rdev.stats.pas_ofld_conn_fails++; 2827 mutex_unlock(&dev->rdev.stats.lock); 2828 kfree_skb(rpl_skb); 2829 } else { 2830 cpl = (struct cpl_pass_accept_req *)cplhdr(rpl_skb); 2831 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 2832 (__force u32) htonl( 2833 (__force u32) req->tid))); 2834 ret = pass_accept_req(dev, rpl_skb); 2835 if (!ret) 2836 kfree_skb(rpl_skb); 2837 } 2838 return; 2839 } 2840 2841 static int deferred_fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 2842 { 2843 struct cpl_fw6_msg *rpl = cplhdr(skb); 2844 struct cpl_fw6_msg_ofld_connection_wr_rpl *req; 2845 2846 switch (rpl->type) { 2847 case FW6_TYPE_CQE: 2848 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 2849 break; 2850 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 2851 req = (struct cpl_fw6_msg_ofld_connection_wr_rpl *)rpl->data; 2852 switch (req->t_state) { 2853 case TCP_SYN_SENT: 2854 active_ofld_conn_reply(dev, skb, req); 2855 break; 2856 case TCP_SYN_RECV: 2857 passive_ofld_conn_reply(dev, skb, req); 2858 break; 2859 default: 2860 pr_err("%s unexpected ofld conn wr state %d\n", 2861 __func__, req->t_state); 2862 break; 2863 } 2864 break; 2865 } 2866 return 0; 2867 } 2868 2869 static void build_cpl_pass_accept_req(struct sk_buff *skb, int stid , u8 tos) 2870 { 2871 u32 l2info; 2872 u16 vlantag, len, hdr_len; 2873 u8 intf; 2874 struct cpl_rx_pkt *cpl = cplhdr(skb); 2875 struct cpl_pass_accept_req *req; 2876 struct tcp_options_received tmp_opt; 2877 2878 /* Store values from cpl_rx_pkt in temporary location. */ 2879 vlantag = (__force u16) cpl->vlan; 2880 len = (__force u16) cpl->len; 2881 l2info = (__force u32) cpl->l2info; 2882 hdr_len = (__force u16) cpl->hdr_len; 2883 intf = cpl->iff; 2884 2885 __skb_pull(skb, sizeof(*req) + sizeof(struct rss_header)); 2886 2887 /* 2888 * We need to parse the TCP options from SYN packet. 2889 * to generate cpl_pass_accept_req. 2890 */ 2891 memset(&tmp_opt, 0, sizeof(tmp_opt)); 2892 tcp_clear_options(&tmp_opt); 2893 tcp_parse_options(skb, &tmp_opt, NULL, 0, NULL); 2894 2895 req = (struct cpl_pass_accept_req *)__skb_push(skb, sizeof(*req)); 2896 memset(req, 0, sizeof(*req)); 2897 req->l2info = cpu_to_be16(V_SYN_INTF(intf) | 2898 V_SYN_MAC_IDX(G_RX_MACIDX( 2899 (__force int) htonl(l2info))) | 2900 F_SYN_XACT_MATCH); 2901 req->hdr_len = cpu_to_be32(V_SYN_RX_CHAN(G_RX_CHAN( 2902 (__force int) htonl(l2info))) | 2903 V_TCP_HDR_LEN(G_RX_TCPHDR_LEN( 2904 (__force int) htons(hdr_len))) | 2905 V_IP_HDR_LEN(G_RX_IPHDR_LEN( 2906 (__force int) htons(hdr_len))) | 2907 V_ETH_HDR_LEN(G_RX_ETHHDR_LEN( 2908 (__force int) htonl(l2info)))); 2909 req->vlan = (__force __be16) vlantag; 2910 req->len = (__force __be16) len; 2911 req->tos_stid = cpu_to_be32(PASS_OPEN_TID(stid) | 2912 PASS_OPEN_TOS(tos)); 2913 req->tcpopt.mss = htons(tmp_opt.mss_clamp); 2914 if (tmp_opt.wscale_ok) 2915 req->tcpopt.wsf = tmp_opt.snd_wscale; 2916 req->tcpopt.tstamp = tmp_opt.saw_tstamp; 2917 if (tmp_opt.sack_ok) 2918 req->tcpopt.sack = 1; 2919 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 0)); 2920 return; 2921 } 2922 2923 static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, 2924 __be32 laddr, __be16 lport, 2925 __be32 raddr, __be16 rport, 2926 u32 rcv_isn, u32 filter, u16 window, 2927 u32 rss_qid, u8 port_id) 2928 { 2929 struct sk_buff *req_skb; 2930 struct fw_ofld_connection_wr *req; 2931 struct cpl_pass_accept_req *cpl = cplhdr(skb); 2932 2933 req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); 2934 req = (struct fw_ofld_connection_wr *)__skb_put(req_skb, sizeof(*req)); 2935 memset(req, 0, sizeof(*req)); 2936 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL(1)); 2937 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 2938 req->le.version_cpl = htonl(F_FW_OFLD_CONNECTION_WR_CPL); 2939 req->le.filter = (__force __be32) filter; 2940 req->le.lport = lport; 2941 req->le.pport = rport; 2942 req->le.u.ipv4.lip = laddr; 2943 req->le.u.ipv4.pip = raddr; 2944 req->tcb.rcv_nxt = htonl(rcv_isn + 1); 2945 req->tcb.rcv_adv = htons(window); 2946 req->tcb.t_state_to_astid = 2947 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_RECV) | 2948 V_FW_OFLD_CONNECTION_WR_RCV_SCALE(cpl->tcpopt.wsf) | 2949 V_FW_OFLD_CONNECTION_WR_ASTID( 2950 GET_PASS_OPEN_TID(ntohl(cpl->tos_stid)))); 2951 2952 /* 2953 * We store the qid in opt2 which will be used by the firmware 2954 * to send us the wr response. 2955 */ 2956 req->tcb.opt2 = htonl(V_RSS_QUEUE(rss_qid)); 2957 2958 /* 2959 * We initialize the MSS index in TCB to 0xF. 2960 * So that when driver sends cpl_pass_accept_rpl 2961 * TCB picks up the correct value. If this was 0 2962 * TP will ignore any value > 0 for MSS index. 2963 */ 2964 req->tcb.opt0 = cpu_to_be64(V_MSS_IDX(0xF)); 2965 req->cookie = (unsigned long)skb; 2966 2967 set_wr_txq(req_skb, CPL_PRIORITY_CONTROL, port_id); 2968 cxgb4_ofld_send(dev->rdev.lldi.ports[0], req_skb); 2969 } 2970 2971 /* 2972 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt 2973 * messages when a filter is being used instead of server to 2974 * redirect a syn packet. When packets hit filter they are redirected 2975 * to the offload queue and driver tries to establish the connection 2976 * using firmware work request. 2977 */ 2978 static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) 2979 { 2980 int stid; 2981 unsigned int filter; 2982 struct ethhdr *eh = NULL; 2983 struct vlan_ethhdr *vlan_eh = NULL; 2984 struct iphdr *iph; 2985 struct tcphdr *tcph; 2986 struct rss_header *rss = (void *)skb->data; 2987 struct cpl_rx_pkt *cpl = (void *)skb->data; 2988 struct cpl_pass_accept_req *req = (void *)(rss + 1); 2989 struct l2t_entry *e; 2990 struct dst_entry *dst; 2991 struct rtable *rt; 2992 struct c4iw_ep *lep; 2993 u16 window; 2994 struct port_info *pi; 2995 struct net_device *pdev; 2996 u16 rss_qid; 2997 int step; 2998 u32 tx_chan; 2999 struct neighbour *neigh; 3000 3001 /* Drop all non-SYN packets */ 3002 if (!(cpl->l2info & cpu_to_be32(F_RXF_SYN))) 3003 goto reject; 3004 3005 /* 3006 * Drop all packets which did not hit the filter. 3007 * Unlikely to happen. 3008 */ 3009 if (!(rss->filter_hit && rss->filter_tid)) 3010 goto reject; 3011 3012 /* 3013 * Calculate the server tid from filter hit index from cpl_rx_pkt. 3014 */ 3015 stid = (__force int) cpu_to_be32((__force u32) rss->hash_val) 3016 - dev->rdev.lldi.tids->sftid_base 3017 + dev->rdev.lldi.tids->nstids; 3018 3019 lep = (struct c4iw_ep *)lookup_stid(dev->rdev.lldi.tids, stid); 3020 if (!lep) { 3021 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 3022 goto reject; 3023 } 3024 3025 if (G_RX_ETHHDR_LEN(ntohl(cpl->l2info)) == ETH_HLEN) { 3026 eh = (struct ethhdr *)(req + 1); 3027 iph = (struct iphdr *)(eh + 1); 3028 } else { 3029 vlan_eh = (struct vlan_ethhdr *)(req + 1); 3030 iph = (struct iphdr *)(vlan_eh + 1); 3031 skb->vlan_tci = ntohs(cpl->vlan); 3032 } 3033 3034 if (iph->version != 0x4) 3035 goto reject; 3036 3037 tcph = (struct tcphdr *)(iph + 1); 3038 skb_set_network_header(skb, (void *)iph - (void *)rss); 3039 skb_set_transport_header(skb, (void *)tcph - (void *)rss); 3040 skb_get(skb); 3041 3042 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__, 3043 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3044 ntohs(tcph->source), iph->tos); 3045 3046 rt = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3047 iph->tos); 3048 if (!rt) { 3049 pr_err("%s - failed to find dst entry!\n", 3050 __func__); 3051 goto reject; 3052 } 3053 dst = &rt->dst; 3054 neigh = dst_neigh_lookup_skb(dst, skb); 3055 3056 if (neigh->dev->flags & IFF_LOOPBACK) { 3057 pdev = ip_dev_find(&init_net, iph->daddr); 3058 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3059 pdev, 0); 3060 pi = (struct port_info *)netdev_priv(pdev); 3061 tx_chan = cxgb4_port_chan(pdev); 3062 dev_put(pdev); 3063 } else { 3064 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3065 neigh->dev, 0); 3066 pi = (struct port_info *)netdev_priv(neigh->dev); 3067 tx_chan = cxgb4_port_chan(neigh->dev); 3068 } 3069 if (!e) { 3070 pr_err("%s - failed to allocate l2t entry!\n", 3071 __func__); 3072 goto free_dst; 3073 } 3074 3075 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; 3076 rss_qid = dev->rdev.lldi.rxq_ids[pi->port_id * step]; 3077 window = (__force u16) htons((__force u16)tcph->window); 3078 3079 /* Calcuate filter portion for LE region. */ 3080 filter = (__force unsigned int) cpu_to_be32(select_ntuple(dev, dst, e)); 3081 3082 /* 3083 * Synthesize the cpl_pass_accept_req. We have everything except the 3084 * TID. Once firmware sends a reply with TID we update the TID field 3085 * in cpl and pass it through the regular cpl_pass_accept_req path. 3086 */ 3087 build_cpl_pass_accept_req(skb, stid, iph->tos); 3088 send_fw_pass_open_req(dev, skb, iph->daddr, tcph->dest, iph->saddr, 3089 tcph->source, ntohl(tcph->seq), filter, window, 3090 rss_qid, pi->port_id); 3091 cxgb4_l2t_release(e); 3092 free_dst: 3093 dst_release(dst); 3094 reject: 3095 return 0; 3096 } 3097 3098 /* 3099 * These are the real handlers that are called from a 3100 * work queue. 3101 */ 3102 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 3103 [CPL_ACT_ESTABLISH] = act_establish, 3104 [CPL_ACT_OPEN_RPL] = act_open_rpl, 3105 [CPL_RX_DATA] = rx_data, 3106 [CPL_ABORT_RPL_RSS] = abort_rpl, 3107 [CPL_ABORT_RPL] = abort_rpl, 3108 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 3109 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 3110 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 3111 [CPL_PASS_ESTABLISH] = pass_establish, 3112 [CPL_PEER_CLOSE] = peer_close, 3113 [CPL_ABORT_REQ_RSS] = peer_abort, 3114 [CPL_CLOSE_CON_RPL] = close_con_rpl, 3115 [CPL_RDMA_TERMINATE] = terminate, 3116 [CPL_FW4_ACK] = fw4_ack, 3117 [CPL_FW6_MSG] = deferred_fw6_msg, 3118 [CPL_RX_PKT] = rx_pkt 3119 }; 3120 3121 static void process_timeout(struct c4iw_ep *ep) 3122 { 3123 struct c4iw_qp_attributes attrs; 3124 int abort = 1; 3125 3126 mutex_lock(&ep->com.mutex); 3127 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 3128 ep->com.state); 3129 set_bit(TIMEDOUT, &ep->com.history); 3130 switch (ep->com.state) { 3131 case MPA_REQ_SENT: 3132 __state_set(&ep->com, ABORTING); 3133 connect_reply_upcall(ep, -ETIMEDOUT); 3134 break; 3135 case MPA_REQ_WAIT: 3136 __state_set(&ep->com, ABORTING); 3137 break; 3138 case CLOSING: 3139 case MORIBUND: 3140 if (ep->com.cm_id && ep->com.qp) { 3141 attrs.next_state = C4IW_QP_STATE_ERROR; 3142 c4iw_modify_qp(ep->com.qp->rhp, 3143 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 3144 &attrs, 1); 3145 } 3146 __state_set(&ep->com, ABORTING); 3147 break; 3148 default: 3149 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3150 __func__, ep, ep->hwtid, ep->com.state); 3151 abort = 0; 3152 } 3153 mutex_unlock(&ep->com.mutex); 3154 if (abort) 3155 abort_connection(ep, NULL, GFP_KERNEL); 3156 c4iw_put_ep(&ep->com); 3157 } 3158 3159 static void process_timedout_eps(void) 3160 { 3161 struct c4iw_ep *ep; 3162 3163 spin_lock_irq(&timeout_lock); 3164 while (!list_empty(&timeout_list)) { 3165 struct list_head *tmp; 3166 3167 tmp = timeout_list.next; 3168 list_del(tmp); 3169 spin_unlock_irq(&timeout_lock); 3170 ep = list_entry(tmp, struct c4iw_ep, entry); 3171 process_timeout(ep); 3172 spin_lock_irq(&timeout_lock); 3173 } 3174 spin_unlock_irq(&timeout_lock); 3175 } 3176 3177 static void process_work(struct work_struct *work) 3178 { 3179 struct sk_buff *skb = NULL; 3180 struct c4iw_dev *dev; 3181 struct cpl_act_establish *rpl; 3182 unsigned int opcode; 3183 int ret; 3184 3185 while ((skb = skb_dequeue(&rxq))) { 3186 rpl = cplhdr(skb); 3187 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3188 opcode = rpl->ot.opcode; 3189 3190 BUG_ON(!work_handlers[opcode]); 3191 ret = work_handlers[opcode](dev, skb); 3192 if (!ret) 3193 kfree_skb(skb); 3194 } 3195 process_timedout_eps(); 3196 } 3197 3198 static DECLARE_WORK(skb_work, process_work); 3199 3200 static void ep_timeout(unsigned long arg) 3201 { 3202 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 3203 int kickit = 0; 3204 3205 spin_lock(&timeout_lock); 3206 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 3207 list_add_tail(&ep->entry, &timeout_list); 3208 kickit = 1; 3209 } 3210 spin_unlock(&timeout_lock); 3211 if (kickit) 3212 queue_work(workq, &skb_work); 3213 } 3214 3215 /* 3216 * All the CM events are handled on a work queue to have a safe context. 3217 */ 3218 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 3219 { 3220 3221 /* 3222 * Save dev in the skb->cb area. 3223 */ 3224 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 3225 3226 /* 3227 * Queue the skb and schedule the worker thread. 3228 */ 3229 skb_queue_tail(&rxq, skb); 3230 queue_work(workq, &skb_work); 3231 return 0; 3232 } 3233 3234 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 3235 { 3236 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 3237 3238 if (rpl->status != CPL_ERR_NONE) { 3239 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 3240 "for tid %u\n", rpl->status, GET_TID(rpl)); 3241 } 3242 kfree_skb(skb); 3243 return 0; 3244 } 3245 3246 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3247 { 3248 struct cpl_fw6_msg *rpl = cplhdr(skb); 3249 struct c4iw_wr_wait *wr_waitp; 3250 int ret; 3251 3252 PDBG("%s type %u\n", __func__, rpl->type); 3253 3254 switch (rpl->type) { 3255 case FW6_TYPE_WR_RPL: 3256 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 3257 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 3258 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 3259 if (wr_waitp) 3260 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 3261 kfree_skb(skb); 3262 break; 3263 case FW6_TYPE_CQE: 3264 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3265 sched(dev, skb); 3266 break; 3267 default: 3268 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 3269 rpl->type); 3270 kfree_skb(skb); 3271 break; 3272 } 3273 return 0; 3274 } 3275 3276 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 3277 { 3278 struct cpl_abort_req_rss *req = cplhdr(skb); 3279 struct c4iw_ep *ep; 3280 struct tid_info *t = dev->rdev.lldi.tids; 3281 unsigned int tid = GET_TID(req); 3282 3283 ep = lookup_tid(t, tid); 3284 if (!ep) { 3285 printk(KERN_WARNING MOD 3286 "Abort on non-existent endpoint, tid %d\n", tid); 3287 kfree_skb(skb); 3288 return 0; 3289 } 3290 if (is_neg_adv_abort(req->status)) { 3291 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 3292 ep->hwtid); 3293 kfree_skb(skb); 3294 return 0; 3295 } 3296 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 3297 ep->com.state); 3298 3299 /* 3300 * Wake up any threads in rdma_init() or rdma_fini(). 3301 * However, if we are on MPAv2 and want to retry with MPAv1 3302 * then, don't wake up yet. 3303 */ 3304 if (mpa_rev == 2 && !ep->tried_with_mpa_v1) { 3305 if (ep->com.state != MPA_REQ_SENT) 3306 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3307 } else 3308 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3309 sched(dev, skb); 3310 return 0; 3311 } 3312 3313 /* 3314 * Most upcalls from the T4 Core go to sched() to 3315 * schedule the processing on a work queue. 3316 */ 3317 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 3318 [CPL_ACT_ESTABLISH] = sched, 3319 [CPL_ACT_OPEN_RPL] = sched, 3320 [CPL_RX_DATA] = sched, 3321 [CPL_ABORT_RPL_RSS] = sched, 3322 [CPL_ABORT_RPL] = sched, 3323 [CPL_PASS_OPEN_RPL] = sched, 3324 [CPL_CLOSE_LISTSRV_RPL] = sched, 3325 [CPL_PASS_ACCEPT_REQ] = sched, 3326 [CPL_PASS_ESTABLISH] = sched, 3327 [CPL_PEER_CLOSE] = sched, 3328 [CPL_CLOSE_CON_RPL] = sched, 3329 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 3330 [CPL_RDMA_TERMINATE] = sched, 3331 [CPL_FW4_ACK] = sched, 3332 [CPL_SET_TCB_RPL] = set_tcb_rpl, 3333 [CPL_FW6_MSG] = fw6_msg, 3334 [CPL_RX_PKT] = sched 3335 }; 3336 3337 int __init c4iw_cm_init(void) 3338 { 3339 spin_lock_init(&timeout_lock); 3340 skb_queue_head_init(&rxq); 3341 3342 workq = create_singlethread_workqueue("iw_cxgb4"); 3343 if (!workq) 3344 return -ENOMEM; 3345 3346 return 0; 3347 } 3348 3349 void __exit c4iw_cm_term(void) 3350 { 3351 WARN_ON(!list_empty(&timeout_list)); 3352 flush_workqueue(workq); 3353 destroy_workqueue(workq); 3354 } 3355