1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 #include <linux/if_vlan.h> 42 43 #include <net/neighbour.h> 44 #include <net/netevent.h> 45 #include <net/route.h> 46 #include <net/tcp.h> 47 48 #include "iw_cxgb4.h" 49 50 static char *states[] = { 51 "idle", 52 "listen", 53 "connecting", 54 "mpa_wait_req", 55 "mpa_req_sent", 56 "mpa_req_rcvd", 57 "mpa_rep_sent", 58 "fpdu_mode", 59 "aborting", 60 "closing", 61 "moribund", 62 "dead", 63 NULL, 64 }; 65 66 static int nocong; 67 module_param(nocong, int, 0644); 68 MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)"); 69 70 static int enable_ecn; 71 module_param(enable_ecn, int, 0644); 72 MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); 73 74 static int dack_mode = 1; 75 module_param(dack_mode, int, 0644); 76 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 77 78 int c4iw_max_read_depth = 8; 79 module_param(c4iw_max_read_depth, int, 0644); 80 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 81 82 static int enable_tcp_timestamps; 83 module_param(enable_tcp_timestamps, int, 0644); 84 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 85 86 static int enable_tcp_sack; 87 module_param(enable_tcp_sack, int, 0644); 88 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 89 90 static int enable_tcp_window_scaling = 1; 91 module_param(enable_tcp_window_scaling, int, 0644); 92 MODULE_PARM_DESC(enable_tcp_window_scaling, 93 "Enable tcp window scaling (default=1)"); 94 95 int c4iw_debug; 96 module_param(c4iw_debug, int, 0644); 97 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 98 99 static int peer2peer; 100 module_param(peer2peer, int, 0644); 101 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 102 103 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 104 module_param(p2p_type, int, 0644); 105 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 106 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 107 108 static int ep_timeout_secs = 60; 109 module_param(ep_timeout_secs, int, 0644); 110 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 111 "in seconds (default=60)"); 112 113 static int mpa_rev = 1; 114 module_param(mpa_rev, int, 0644); 115 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 116 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 117 " compliant (default=1)"); 118 119 static int markers_enabled; 120 module_param(markers_enabled, int, 0644); 121 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 122 123 static int crc_enabled = 1; 124 module_param(crc_enabled, int, 0644); 125 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 126 127 static int rcv_win = 256 * 1024; 128 module_param(rcv_win, int, 0644); 129 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 130 131 static int snd_win = 128 * 1024; 132 module_param(snd_win, int, 0644); 133 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 134 135 static struct workqueue_struct *workq; 136 137 static struct sk_buff_head rxq; 138 139 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 140 static void ep_timeout(unsigned long arg); 141 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 142 143 static LIST_HEAD(timeout_list); 144 static spinlock_t timeout_lock; 145 146 static void start_ep_timer(struct c4iw_ep *ep) 147 { 148 PDBG("%s ep %p\n", __func__, ep); 149 if (timer_pending(&ep->timer)) { 150 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep); 151 del_timer_sync(&ep->timer); 152 } else 153 c4iw_get_ep(&ep->com); 154 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 155 ep->timer.data = (unsigned long)ep; 156 ep->timer.function = ep_timeout; 157 add_timer(&ep->timer); 158 } 159 160 static void stop_ep_timer(struct c4iw_ep *ep) 161 { 162 PDBG("%s ep %p\n", __func__, ep); 163 if (!timer_pending(&ep->timer)) { 164 WARN(1, "%s timer stopped when its not running! " 165 "ep %p state %u\n", __func__, ep, ep->com.state); 166 return; 167 } 168 del_timer_sync(&ep->timer); 169 c4iw_put_ep(&ep->com); 170 } 171 172 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 173 struct l2t_entry *l2e) 174 { 175 int error = 0; 176 177 if (c4iw_fatal_error(rdev)) { 178 kfree_skb(skb); 179 PDBG("%s - device in error state - dropping\n", __func__); 180 return -EIO; 181 } 182 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 183 if (error < 0) 184 kfree_skb(skb); 185 return error < 0 ? error : 0; 186 } 187 188 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 189 { 190 int error = 0; 191 192 if (c4iw_fatal_error(rdev)) { 193 kfree_skb(skb); 194 PDBG("%s - device in error state - dropping\n", __func__); 195 return -EIO; 196 } 197 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 198 if (error < 0) 199 kfree_skb(skb); 200 return error < 0 ? error : 0; 201 } 202 203 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 204 { 205 struct cpl_tid_release *req; 206 207 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 208 if (!skb) 209 return; 210 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 211 INIT_TP_WR(req, hwtid); 212 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 213 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 214 c4iw_ofld_send(rdev, skb); 215 return; 216 } 217 218 static void set_emss(struct c4iw_ep *ep, u16 opt) 219 { 220 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 221 ep->mss = ep->emss; 222 if (GET_TCPOPT_TSTAMP(opt)) 223 ep->emss -= 12; 224 if (ep->emss < 128) 225 ep->emss = 128; 226 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 227 ep->mss, ep->emss); 228 } 229 230 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 231 { 232 enum c4iw_ep_state state; 233 234 mutex_lock(&epc->mutex); 235 state = epc->state; 236 mutex_unlock(&epc->mutex); 237 return state; 238 } 239 240 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 241 { 242 epc->state = new; 243 } 244 245 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 246 { 247 mutex_lock(&epc->mutex); 248 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 249 __state_set(epc, new); 250 mutex_unlock(&epc->mutex); 251 return; 252 } 253 254 static void *alloc_ep(int size, gfp_t gfp) 255 { 256 struct c4iw_ep_common *epc; 257 258 epc = kzalloc(size, gfp); 259 if (epc) { 260 kref_init(&epc->kref); 261 mutex_init(&epc->mutex); 262 c4iw_init_wr_wait(&epc->wr_wait); 263 } 264 PDBG("%s alloc ep %p\n", __func__, epc); 265 return epc; 266 } 267 268 void _c4iw_free_ep(struct kref *kref) 269 { 270 struct c4iw_ep *ep; 271 272 ep = container_of(kref, struct c4iw_ep, com.kref); 273 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 274 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 275 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 276 dst_release(ep->dst); 277 cxgb4_l2t_release(ep->l2t); 278 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 279 } 280 kfree(ep); 281 } 282 283 static void release_ep_resources(struct c4iw_ep *ep) 284 { 285 set_bit(RELEASE_RESOURCES, &ep->com.flags); 286 c4iw_put_ep(&ep->com); 287 } 288 289 static int status2errno(int status) 290 { 291 switch (status) { 292 case CPL_ERR_NONE: 293 return 0; 294 case CPL_ERR_CONN_RESET: 295 return -ECONNRESET; 296 case CPL_ERR_ARP_MISS: 297 return -EHOSTUNREACH; 298 case CPL_ERR_CONN_TIMEDOUT: 299 return -ETIMEDOUT; 300 case CPL_ERR_TCAM_FULL: 301 return -ENOMEM; 302 case CPL_ERR_CONN_EXIST: 303 return -EADDRINUSE; 304 default: 305 return -EIO; 306 } 307 } 308 309 /* 310 * Try and reuse skbs already allocated... 311 */ 312 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 313 { 314 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 315 skb_trim(skb, 0); 316 skb_get(skb); 317 skb_reset_transport_header(skb); 318 } else { 319 skb = alloc_skb(len, gfp); 320 } 321 return skb; 322 } 323 324 static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip, 325 __be32 peer_ip, __be16 local_port, 326 __be16 peer_port, u8 tos) 327 { 328 struct rtable *rt; 329 struct flowi4 fl4; 330 331 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 332 peer_port, local_port, IPPROTO_TCP, 333 tos, 0); 334 if (IS_ERR(rt)) 335 return NULL; 336 return rt; 337 } 338 339 static void arp_failure_discard(void *handle, struct sk_buff *skb) 340 { 341 PDBG("%s c4iw_dev %p\n", __func__, handle); 342 kfree_skb(skb); 343 } 344 345 /* 346 * Handle an ARP failure for an active open. 347 */ 348 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 349 { 350 printk(KERN_ERR MOD "ARP failure duing connect\n"); 351 kfree_skb(skb); 352 } 353 354 /* 355 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 356 * and send it along. 357 */ 358 static void abort_arp_failure(void *handle, struct sk_buff *skb) 359 { 360 struct c4iw_rdev *rdev = handle; 361 struct cpl_abort_req *req = cplhdr(skb); 362 363 PDBG("%s rdev %p\n", __func__, rdev); 364 req->cmd = CPL_ABORT_NO_RST; 365 c4iw_ofld_send(rdev, skb); 366 } 367 368 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 369 { 370 unsigned int flowclen = 80; 371 struct fw_flowc_wr *flowc; 372 int i; 373 374 skb = get_skb(skb, flowclen, GFP_KERNEL); 375 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 376 377 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 378 FW_FLOWC_WR_NPARAMS(8)); 379 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 380 16)) | FW_WR_FLOWID(ep->hwtid)); 381 382 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 383 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 384 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 385 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 386 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 387 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 388 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 389 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 390 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 391 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 392 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 393 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 394 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 395 flowc->mnemval[6].val = cpu_to_be32(snd_win); 396 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 397 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 398 /* Pad WR to 16 byte boundary */ 399 flowc->mnemval[8].mnemonic = 0; 400 flowc->mnemval[8].val = 0; 401 for (i = 0; i < 9; i++) { 402 flowc->mnemval[i].r4[0] = 0; 403 flowc->mnemval[i].r4[1] = 0; 404 flowc->mnemval[i].r4[2] = 0; 405 } 406 407 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 408 c4iw_ofld_send(&ep->com.dev->rdev, skb); 409 } 410 411 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 412 { 413 struct cpl_close_con_req *req; 414 struct sk_buff *skb; 415 int wrlen = roundup(sizeof *req, 16); 416 417 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 418 skb = get_skb(NULL, wrlen, gfp); 419 if (!skb) { 420 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 421 return -ENOMEM; 422 } 423 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 424 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 425 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 426 memset(req, 0, wrlen); 427 INIT_TP_WR(req, ep->hwtid); 428 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 429 ep->hwtid)); 430 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 431 } 432 433 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 434 { 435 struct cpl_abort_req *req; 436 int wrlen = roundup(sizeof *req, 16); 437 438 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 439 skb = get_skb(skb, wrlen, gfp); 440 if (!skb) { 441 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 442 __func__); 443 return -ENOMEM; 444 } 445 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 446 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 447 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 448 memset(req, 0, wrlen); 449 INIT_TP_WR(req, ep->hwtid); 450 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 451 req->cmd = CPL_ABORT_SEND_RST; 452 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 453 } 454 455 #define VLAN_NONE 0xfff 456 #define FILTER_SEL_VLAN_NONE 0xffff 457 #define FILTER_SEL_WIDTH_P_FC (3+1) /* port uses 3 bits, FCoE one bit */ 458 #define FILTER_SEL_WIDTH_VIN_P_FC \ 459 (6 + 7 + FILTER_SEL_WIDTH_P_FC) /* 6 bits are unused, VF uses 7 bits*/ 460 #define FILTER_SEL_WIDTH_TAG_P_FC \ 461 (3 + FILTER_SEL_WIDTH_VIN_P_FC) /* PF uses 3 bits */ 462 #define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC) 463 464 static unsigned int select_ntuple(struct c4iw_dev *dev, struct dst_entry *dst, 465 struct l2t_entry *l2t) 466 { 467 unsigned int ntuple = 0; 468 u32 viid; 469 470 switch (dev->rdev.lldi.filt_mode) { 471 472 /* default filter mode */ 473 case HW_TPL_FR_MT_PR_IV_P_FC: 474 if (l2t->vlan == VLAN_NONE) 475 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 476 else { 477 ntuple |= l2t->vlan << FILTER_SEL_WIDTH_P_FC; 478 ntuple |= 1 << FILTER_SEL_WIDTH_VLD_TAG_P_FC; 479 } 480 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 481 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 482 break; 483 case HW_TPL_FR_MT_PR_OV_P_FC: { 484 viid = cxgb4_port_viid(l2t->neigh->dev); 485 486 ntuple |= FW_VIID_VIN_GET(viid) << FILTER_SEL_WIDTH_P_FC; 487 ntuple |= FW_VIID_PFN_GET(viid) << FILTER_SEL_WIDTH_VIN_P_FC; 488 ntuple |= FW_VIID_VIVLD_GET(viid) << FILTER_SEL_WIDTH_TAG_P_FC; 489 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 490 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 491 break; 492 } 493 default: 494 break; 495 } 496 return ntuple; 497 } 498 499 static int send_connect(struct c4iw_ep *ep) 500 { 501 struct cpl_act_open_req *req; 502 struct sk_buff *skb; 503 u64 opt0; 504 u32 opt2; 505 unsigned int mtu_idx; 506 int wscale; 507 int wrlen = roundup(sizeof *req, 16); 508 509 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 510 511 skb = get_skb(NULL, wrlen, GFP_KERNEL); 512 if (!skb) { 513 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 514 __func__); 515 return -ENOMEM; 516 } 517 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 518 519 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 520 wscale = compute_wscale(rcv_win); 521 opt0 = (nocong ? NO_CONG(1) : 0) | 522 KEEP_ALIVE(1) | 523 DELACK(1) | 524 WND_SCALE(wscale) | 525 MSS_IDX(mtu_idx) | 526 L2T_IDX(ep->l2t->idx) | 527 TX_CHAN(ep->tx_chan) | 528 SMAC_SEL(ep->smac_idx) | 529 DSCP(ep->tos) | 530 ULP_MODE(ULP_MODE_TCPDDP) | 531 RCV_BUFSIZ(rcv_win>>10); 532 opt2 = RX_CHANNEL(0) | 533 CCTRL_ECN(enable_ecn) | 534 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 535 if (enable_tcp_timestamps) 536 opt2 |= TSTAMPS_EN(1); 537 if (enable_tcp_sack) 538 opt2 |= SACK_EN(1); 539 if (wscale && enable_tcp_window_scaling) 540 opt2 |= WND_SCALE_EN(1); 541 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 542 543 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 544 INIT_TP_WR(req, 0); 545 OPCODE_TID(req) = cpu_to_be32( 546 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid))); 547 req->local_port = ep->com.local_addr.sin_port; 548 req->peer_port = ep->com.remote_addr.sin_port; 549 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 550 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 551 req->opt0 = cpu_to_be64(opt0); 552 req->params = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, ep->l2t)); 553 req->opt2 = cpu_to_be32(opt2); 554 set_bit(ACT_OPEN_REQ, &ep->com.history); 555 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 556 } 557 558 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 559 u8 mpa_rev_to_use) 560 { 561 int mpalen, wrlen; 562 struct fw_ofld_tx_data_wr *req; 563 struct mpa_message *mpa; 564 struct mpa_v2_conn_params mpa_v2_params; 565 566 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 567 568 BUG_ON(skb_cloned(skb)); 569 570 mpalen = sizeof(*mpa) + ep->plen; 571 if (mpa_rev_to_use == 2) 572 mpalen += sizeof(struct mpa_v2_conn_params); 573 wrlen = roundup(mpalen + sizeof *req, 16); 574 skb = get_skb(skb, wrlen, GFP_KERNEL); 575 if (!skb) { 576 connect_reply_upcall(ep, -ENOMEM); 577 return; 578 } 579 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 580 581 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 582 memset(req, 0, wrlen); 583 req->op_to_immdlen = cpu_to_be32( 584 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 585 FW_WR_COMPL(1) | 586 FW_WR_IMMDLEN(mpalen)); 587 req->flowid_len16 = cpu_to_be32( 588 FW_WR_FLOWID(ep->hwtid) | 589 FW_WR_LEN16(wrlen >> 4)); 590 req->plen = cpu_to_be32(mpalen); 591 req->tunnel_to_proxy = cpu_to_be32( 592 FW_OFLD_TX_DATA_WR_FLUSH(1) | 593 FW_OFLD_TX_DATA_WR_SHOVE(1)); 594 595 mpa = (struct mpa_message *)(req + 1); 596 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 597 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 598 (markers_enabled ? MPA_MARKERS : 0) | 599 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 600 mpa->private_data_size = htons(ep->plen); 601 mpa->revision = mpa_rev_to_use; 602 if (mpa_rev_to_use == 1) { 603 ep->tried_with_mpa_v1 = 1; 604 ep->retry_with_mpa_v1 = 0; 605 } 606 607 if (mpa_rev_to_use == 2) { 608 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 609 sizeof (struct mpa_v2_conn_params)); 610 mpa_v2_params.ird = htons((u16)ep->ird); 611 mpa_v2_params.ord = htons((u16)ep->ord); 612 613 if (peer2peer) { 614 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 615 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 616 mpa_v2_params.ord |= 617 htons(MPA_V2_RDMA_WRITE_RTR); 618 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 619 mpa_v2_params.ord |= 620 htons(MPA_V2_RDMA_READ_RTR); 621 } 622 memcpy(mpa->private_data, &mpa_v2_params, 623 sizeof(struct mpa_v2_conn_params)); 624 625 if (ep->plen) 626 memcpy(mpa->private_data + 627 sizeof(struct mpa_v2_conn_params), 628 ep->mpa_pkt + sizeof(*mpa), ep->plen); 629 } else 630 if (ep->plen) 631 memcpy(mpa->private_data, 632 ep->mpa_pkt + sizeof(*mpa), ep->plen); 633 634 /* 635 * Reference the mpa skb. This ensures the data area 636 * will remain in memory until the hw acks the tx. 637 * Function fw4_ack() will deref it. 638 */ 639 skb_get(skb); 640 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 641 BUG_ON(ep->mpa_skb); 642 ep->mpa_skb = skb; 643 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 644 start_ep_timer(ep); 645 state_set(&ep->com, MPA_REQ_SENT); 646 ep->mpa_attr.initiator = 1; 647 return; 648 } 649 650 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 651 { 652 int mpalen, wrlen; 653 struct fw_ofld_tx_data_wr *req; 654 struct mpa_message *mpa; 655 struct sk_buff *skb; 656 struct mpa_v2_conn_params mpa_v2_params; 657 658 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 659 660 mpalen = sizeof(*mpa) + plen; 661 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 662 mpalen += sizeof(struct mpa_v2_conn_params); 663 wrlen = roundup(mpalen + sizeof *req, 16); 664 665 skb = get_skb(NULL, wrlen, GFP_KERNEL); 666 if (!skb) { 667 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 668 return -ENOMEM; 669 } 670 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 671 672 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 673 memset(req, 0, wrlen); 674 req->op_to_immdlen = cpu_to_be32( 675 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 676 FW_WR_COMPL(1) | 677 FW_WR_IMMDLEN(mpalen)); 678 req->flowid_len16 = cpu_to_be32( 679 FW_WR_FLOWID(ep->hwtid) | 680 FW_WR_LEN16(wrlen >> 4)); 681 req->plen = cpu_to_be32(mpalen); 682 req->tunnel_to_proxy = cpu_to_be32( 683 FW_OFLD_TX_DATA_WR_FLUSH(1) | 684 FW_OFLD_TX_DATA_WR_SHOVE(1)); 685 686 mpa = (struct mpa_message *)(req + 1); 687 memset(mpa, 0, sizeof(*mpa)); 688 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 689 mpa->flags = MPA_REJECT; 690 mpa->revision = mpa_rev; 691 mpa->private_data_size = htons(plen); 692 693 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 694 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 695 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 696 sizeof (struct mpa_v2_conn_params)); 697 mpa_v2_params.ird = htons(((u16)ep->ird) | 698 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 699 0)); 700 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 701 (p2p_type == 702 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 703 MPA_V2_RDMA_WRITE_RTR : p2p_type == 704 FW_RI_INIT_P2PTYPE_READ_REQ ? 705 MPA_V2_RDMA_READ_RTR : 0) : 0)); 706 memcpy(mpa->private_data, &mpa_v2_params, 707 sizeof(struct mpa_v2_conn_params)); 708 709 if (ep->plen) 710 memcpy(mpa->private_data + 711 sizeof(struct mpa_v2_conn_params), pdata, plen); 712 } else 713 if (plen) 714 memcpy(mpa->private_data, pdata, plen); 715 716 /* 717 * Reference the mpa skb again. This ensures the data area 718 * will remain in memory until the hw acks the tx. 719 * Function fw4_ack() will deref it. 720 */ 721 skb_get(skb); 722 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 723 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 724 BUG_ON(ep->mpa_skb); 725 ep->mpa_skb = skb; 726 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 727 } 728 729 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 730 { 731 int mpalen, wrlen; 732 struct fw_ofld_tx_data_wr *req; 733 struct mpa_message *mpa; 734 struct sk_buff *skb; 735 struct mpa_v2_conn_params mpa_v2_params; 736 737 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 738 739 mpalen = sizeof(*mpa) + plen; 740 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 741 mpalen += sizeof(struct mpa_v2_conn_params); 742 wrlen = roundup(mpalen + sizeof *req, 16); 743 744 skb = get_skb(NULL, wrlen, GFP_KERNEL); 745 if (!skb) { 746 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 747 return -ENOMEM; 748 } 749 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 750 751 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 752 memset(req, 0, wrlen); 753 req->op_to_immdlen = cpu_to_be32( 754 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 755 FW_WR_COMPL(1) | 756 FW_WR_IMMDLEN(mpalen)); 757 req->flowid_len16 = cpu_to_be32( 758 FW_WR_FLOWID(ep->hwtid) | 759 FW_WR_LEN16(wrlen >> 4)); 760 req->plen = cpu_to_be32(mpalen); 761 req->tunnel_to_proxy = cpu_to_be32( 762 FW_OFLD_TX_DATA_WR_FLUSH(1) | 763 FW_OFLD_TX_DATA_WR_SHOVE(1)); 764 765 mpa = (struct mpa_message *)(req + 1); 766 memset(mpa, 0, sizeof(*mpa)); 767 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 768 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 769 (markers_enabled ? MPA_MARKERS : 0); 770 mpa->revision = ep->mpa_attr.version; 771 mpa->private_data_size = htons(plen); 772 773 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 774 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 775 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 776 sizeof (struct mpa_v2_conn_params)); 777 mpa_v2_params.ird = htons((u16)ep->ird); 778 mpa_v2_params.ord = htons((u16)ep->ord); 779 if (peer2peer && (ep->mpa_attr.p2p_type != 780 FW_RI_INIT_P2PTYPE_DISABLED)) { 781 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 782 783 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 784 mpa_v2_params.ord |= 785 htons(MPA_V2_RDMA_WRITE_RTR); 786 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 787 mpa_v2_params.ord |= 788 htons(MPA_V2_RDMA_READ_RTR); 789 } 790 791 memcpy(mpa->private_data, &mpa_v2_params, 792 sizeof(struct mpa_v2_conn_params)); 793 794 if (ep->plen) 795 memcpy(mpa->private_data + 796 sizeof(struct mpa_v2_conn_params), pdata, plen); 797 } else 798 if (plen) 799 memcpy(mpa->private_data, pdata, plen); 800 801 /* 802 * Reference the mpa skb. This ensures the data area 803 * will remain in memory until the hw acks the tx. 804 * Function fw4_ack() will deref it. 805 */ 806 skb_get(skb); 807 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 808 ep->mpa_skb = skb; 809 state_set(&ep->com, MPA_REP_SENT); 810 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 811 } 812 813 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 814 { 815 struct c4iw_ep *ep; 816 struct cpl_act_establish *req = cplhdr(skb); 817 unsigned int tid = GET_TID(req); 818 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 819 struct tid_info *t = dev->rdev.lldi.tids; 820 821 ep = lookup_atid(t, atid); 822 823 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 824 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 825 826 dst_confirm(ep->dst); 827 828 /* setup the hwtid for this connection */ 829 ep->hwtid = tid; 830 cxgb4_insert_tid(t, ep, tid); 831 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 832 833 ep->snd_seq = be32_to_cpu(req->snd_isn); 834 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 835 836 set_emss(ep, ntohs(req->tcp_opt)); 837 838 /* dealloc the atid */ 839 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 840 cxgb4_free_atid(t, atid); 841 set_bit(ACT_ESTAB, &ep->com.history); 842 843 /* start MPA negotiation */ 844 send_flowc(ep, NULL); 845 if (ep->retry_with_mpa_v1) 846 send_mpa_req(ep, skb, 1); 847 else 848 send_mpa_req(ep, skb, mpa_rev); 849 850 return 0; 851 } 852 853 static void close_complete_upcall(struct c4iw_ep *ep) 854 { 855 struct iw_cm_event event; 856 857 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 858 memset(&event, 0, sizeof(event)); 859 event.event = IW_CM_EVENT_CLOSE; 860 if (ep->com.cm_id) { 861 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 862 ep, ep->com.cm_id, ep->hwtid); 863 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 864 ep->com.cm_id->rem_ref(ep->com.cm_id); 865 ep->com.cm_id = NULL; 866 ep->com.qp = NULL; 867 set_bit(CLOSE_UPCALL, &ep->com.history); 868 } 869 } 870 871 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 872 { 873 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 874 close_complete_upcall(ep); 875 state_set(&ep->com, ABORTING); 876 set_bit(ABORT_CONN, &ep->com.history); 877 return send_abort(ep, skb, gfp); 878 } 879 880 static void peer_close_upcall(struct c4iw_ep *ep) 881 { 882 struct iw_cm_event event; 883 884 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 885 memset(&event, 0, sizeof(event)); 886 event.event = IW_CM_EVENT_DISCONNECT; 887 if (ep->com.cm_id) { 888 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 889 ep, ep->com.cm_id, ep->hwtid); 890 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 891 set_bit(DISCONN_UPCALL, &ep->com.history); 892 } 893 } 894 895 static void peer_abort_upcall(struct c4iw_ep *ep) 896 { 897 struct iw_cm_event event; 898 899 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 900 memset(&event, 0, sizeof(event)); 901 event.event = IW_CM_EVENT_CLOSE; 902 event.status = -ECONNRESET; 903 if (ep->com.cm_id) { 904 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 905 ep->com.cm_id, ep->hwtid); 906 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 907 ep->com.cm_id->rem_ref(ep->com.cm_id); 908 ep->com.cm_id = NULL; 909 ep->com.qp = NULL; 910 set_bit(ABORT_UPCALL, &ep->com.history); 911 } 912 } 913 914 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 915 { 916 struct iw_cm_event event; 917 918 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 919 memset(&event, 0, sizeof(event)); 920 event.event = IW_CM_EVENT_CONNECT_REPLY; 921 event.status = status; 922 event.local_addr = ep->com.local_addr; 923 event.remote_addr = ep->com.remote_addr; 924 925 if ((status == 0) || (status == -ECONNREFUSED)) { 926 if (!ep->tried_with_mpa_v1) { 927 /* this means MPA_v2 is used */ 928 event.private_data_len = ep->plen - 929 sizeof(struct mpa_v2_conn_params); 930 event.private_data = ep->mpa_pkt + 931 sizeof(struct mpa_message) + 932 sizeof(struct mpa_v2_conn_params); 933 } else { 934 /* this means MPA_v1 is used */ 935 event.private_data_len = ep->plen; 936 event.private_data = ep->mpa_pkt + 937 sizeof(struct mpa_message); 938 } 939 } 940 941 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 942 ep->hwtid, status); 943 set_bit(CONN_RPL_UPCALL, &ep->com.history); 944 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 945 946 if (status < 0) { 947 ep->com.cm_id->rem_ref(ep->com.cm_id); 948 ep->com.cm_id = NULL; 949 ep->com.qp = NULL; 950 } 951 } 952 953 static void connect_request_upcall(struct c4iw_ep *ep) 954 { 955 struct iw_cm_event event; 956 957 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 958 memset(&event, 0, sizeof(event)); 959 event.event = IW_CM_EVENT_CONNECT_REQUEST; 960 event.local_addr = ep->com.local_addr; 961 event.remote_addr = ep->com.remote_addr; 962 event.provider_data = ep; 963 if (!ep->tried_with_mpa_v1) { 964 /* this means MPA_v2 is used */ 965 event.ord = ep->ord; 966 event.ird = ep->ird; 967 event.private_data_len = ep->plen - 968 sizeof(struct mpa_v2_conn_params); 969 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 970 sizeof(struct mpa_v2_conn_params); 971 } else { 972 /* this means MPA_v1 is used. Send max supported */ 973 event.ord = c4iw_max_read_depth; 974 event.ird = c4iw_max_read_depth; 975 event.private_data_len = ep->plen; 976 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 977 } 978 if (state_read(&ep->parent_ep->com) != DEAD) { 979 c4iw_get_ep(&ep->com); 980 ep->parent_ep->com.cm_id->event_handler( 981 ep->parent_ep->com.cm_id, 982 &event); 983 } 984 set_bit(CONNREQ_UPCALL, &ep->com.history); 985 c4iw_put_ep(&ep->parent_ep->com); 986 ep->parent_ep = NULL; 987 } 988 989 static void established_upcall(struct c4iw_ep *ep) 990 { 991 struct iw_cm_event event; 992 993 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 994 memset(&event, 0, sizeof(event)); 995 event.event = IW_CM_EVENT_ESTABLISHED; 996 event.ird = ep->ird; 997 event.ord = ep->ord; 998 if (ep->com.cm_id) { 999 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1000 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1001 set_bit(ESTAB_UPCALL, &ep->com.history); 1002 } 1003 } 1004 1005 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 1006 { 1007 struct cpl_rx_data_ack *req; 1008 struct sk_buff *skb; 1009 int wrlen = roundup(sizeof *req, 16); 1010 1011 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 1012 skb = get_skb(NULL, wrlen, GFP_KERNEL); 1013 if (!skb) { 1014 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 1015 return 0; 1016 } 1017 1018 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 1019 memset(req, 0, wrlen); 1020 INIT_TP_WR(req, ep->hwtid); 1021 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1022 ep->hwtid)); 1023 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 1024 F_RX_DACK_CHANGE | 1025 V_RX_DACK_MODE(dack_mode)); 1026 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 1027 c4iw_ofld_send(&ep->com.dev->rdev, skb); 1028 return credits; 1029 } 1030 1031 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 1032 { 1033 struct mpa_message *mpa; 1034 struct mpa_v2_conn_params *mpa_v2_params; 1035 u16 plen; 1036 u16 resp_ird, resp_ord; 1037 u8 rtr_mismatch = 0, insuff_ird = 0; 1038 struct c4iw_qp_attributes attrs; 1039 enum c4iw_qp_attr_mask mask; 1040 int err; 1041 1042 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1043 1044 /* 1045 * Stop mpa timer. If it expired, then the state has 1046 * changed and we bail since ep_timeout already aborted 1047 * the connection. 1048 */ 1049 stop_ep_timer(ep); 1050 if (state_read(&ep->com) != MPA_REQ_SENT) 1051 return; 1052 1053 /* 1054 * If we get more than the supported amount of private data 1055 * then we must fail this connection. 1056 */ 1057 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1058 err = -EINVAL; 1059 goto err; 1060 } 1061 1062 /* 1063 * copy the new data into our accumulation buffer. 1064 */ 1065 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1066 skb->len); 1067 ep->mpa_pkt_len += skb->len; 1068 1069 /* 1070 * if we don't even have the mpa message, then bail. 1071 */ 1072 if (ep->mpa_pkt_len < sizeof(*mpa)) 1073 return; 1074 mpa = (struct mpa_message *) ep->mpa_pkt; 1075 1076 /* Validate MPA header. */ 1077 if (mpa->revision > mpa_rev) { 1078 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1079 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1080 err = -EPROTO; 1081 goto err; 1082 } 1083 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1084 err = -EPROTO; 1085 goto err; 1086 } 1087 1088 plen = ntohs(mpa->private_data_size); 1089 1090 /* 1091 * Fail if there's too much private data. 1092 */ 1093 if (plen > MPA_MAX_PRIVATE_DATA) { 1094 err = -EPROTO; 1095 goto err; 1096 } 1097 1098 /* 1099 * If plen does not account for pkt size 1100 */ 1101 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1102 err = -EPROTO; 1103 goto err; 1104 } 1105 1106 ep->plen = (u8) plen; 1107 1108 /* 1109 * If we don't have all the pdata yet, then bail. 1110 * We'll continue process when more data arrives. 1111 */ 1112 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1113 return; 1114 1115 if (mpa->flags & MPA_REJECT) { 1116 err = -ECONNREFUSED; 1117 goto err; 1118 } 1119 1120 /* 1121 * If we get here we have accumulated the entire mpa 1122 * start reply message including private data. And 1123 * the MPA header is valid. 1124 */ 1125 state_set(&ep->com, FPDU_MODE); 1126 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1127 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1128 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1129 ep->mpa_attr.version = mpa->revision; 1130 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1131 1132 if (mpa->revision == 2) { 1133 ep->mpa_attr.enhanced_rdma_conn = 1134 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1135 if (ep->mpa_attr.enhanced_rdma_conn) { 1136 mpa_v2_params = (struct mpa_v2_conn_params *) 1137 (ep->mpa_pkt + sizeof(*mpa)); 1138 resp_ird = ntohs(mpa_v2_params->ird) & 1139 MPA_V2_IRD_ORD_MASK; 1140 resp_ord = ntohs(mpa_v2_params->ord) & 1141 MPA_V2_IRD_ORD_MASK; 1142 1143 /* 1144 * This is a double-check. Ideally, below checks are 1145 * not required since ird/ord stuff has been taken 1146 * care of in c4iw_accept_cr 1147 */ 1148 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1149 err = -ENOMEM; 1150 ep->ird = resp_ord; 1151 ep->ord = resp_ird; 1152 insuff_ird = 1; 1153 } 1154 1155 if (ntohs(mpa_v2_params->ird) & 1156 MPA_V2_PEER2PEER_MODEL) { 1157 if (ntohs(mpa_v2_params->ord) & 1158 MPA_V2_RDMA_WRITE_RTR) 1159 ep->mpa_attr.p2p_type = 1160 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1161 else if (ntohs(mpa_v2_params->ord) & 1162 MPA_V2_RDMA_READ_RTR) 1163 ep->mpa_attr.p2p_type = 1164 FW_RI_INIT_P2PTYPE_READ_REQ; 1165 } 1166 } 1167 } else if (mpa->revision == 1) 1168 if (peer2peer) 1169 ep->mpa_attr.p2p_type = p2p_type; 1170 1171 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1172 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1173 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1174 ep->mpa_attr.recv_marker_enabled, 1175 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1176 ep->mpa_attr.p2p_type, p2p_type); 1177 1178 /* 1179 * If responder's RTR does not match with that of initiator, assign 1180 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1181 * generated when moving QP to RTS state. 1182 * A TERM message will be sent after QP has moved to RTS state 1183 */ 1184 if ((ep->mpa_attr.version == 2) && peer2peer && 1185 (ep->mpa_attr.p2p_type != p2p_type)) { 1186 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1187 rtr_mismatch = 1; 1188 } 1189 1190 attrs.mpa_attr = ep->mpa_attr; 1191 attrs.max_ird = ep->ird; 1192 attrs.max_ord = ep->ord; 1193 attrs.llp_stream_handle = ep; 1194 attrs.next_state = C4IW_QP_STATE_RTS; 1195 1196 mask = C4IW_QP_ATTR_NEXT_STATE | 1197 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1198 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1199 1200 /* bind QP and TID with INIT_WR */ 1201 err = c4iw_modify_qp(ep->com.qp->rhp, 1202 ep->com.qp, mask, &attrs, 1); 1203 if (err) 1204 goto err; 1205 1206 /* 1207 * If responder's RTR requirement did not match with what initiator 1208 * supports, generate TERM message 1209 */ 1210 if (rtr_mismatch) { 1211 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1212 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1213 attrs.ecode = MPA_NOMATCH_RTR; 1214 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1215 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1216 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1217 err = -ENOMEM; 1218 goto out; 1219 } 1220 1221 /* 1222 * Generate TERM if initiator IRD is not sufficient for responder 1223 * provided ORD. Currently, we do the same behaviour even when 1224 * responder provided IRD is also not sufficient as regards to 1225 * initiator ORD. 1226 */ 1227 if (insuff_ird) { 1228 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1229 __func__); 1230 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1231 attrs.ecode = MPA_INSUFF_IRD; 1232 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1233 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1234 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1235 err = -ENOMEM; 1236 goto out; 1237 } 1238 goto out; 1239 err: 1240 state_set(&ep->com, ABORTING); 1241 send_abort(ep, skb, GFP_KERNEL); 1242 out: 1243 connect_reply_upcall(ep, err); 1244 return; 1245 } 1246 1247 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1248 { 1249 struct mpa_message *mpa; 1250 struct mpa_v2_conn_params *mpa_v2_params; 1251 u16 plen; 1252 1253 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1254 1255 if (state_read(&ep->com) != MPA_REQ_WAIT) 1256 return; 1257 1258 /* 1259 * If we get more than the supported amount of private data 1260 * then we must fail this connection. 1261 */ 1262 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1263 stop_ep_timer(ep); 1264 abort_connection(ep, skb, GFP_KERNEL); 1265 return; 1266 } 1267 1268 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1269 1270 /* 1271 * Copy the new data into our accumulation buffer. 1272 */ 1273 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1274 skb->len); 1275 ep->mpa_pkt_len += skb->len; 1276 1277 /* 1278 * If we don't even have the mpa message, then bail. 1279 * We'll continue process when more data arrives. 1280 */ 1281 if (ep->mpa_pkt_len < sizeof(*mpa)) 1282 return; 1283 1284 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1285 stop_ep_timer(ep); 1286 mpa = (struct mpa_message *) ep->mpa_pkt; 1287 1288 /* 1289 * Validate MPA Header. 1290 */ 1291 if (mpa->revision > mpa_rev) { 1292 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1293 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1294 abort_connection(ep, skb, GFP_KERNEL); 1295 return; 1296 } 1297 1298 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1299 abort_connection(ep, skb, GFP_KERNEL); 1300 return; 1301 } 1302 1303 plen = ntohs(mpa->private_data_size); 1304 1305 /* 1306 * Fail if there's too much private data. 1307 */ 1308 if (plen > MPA_MAX_PRIVATE_DATA) { 1309 abort_connection(ep, skb, GFP_KERNEL); 1310 return; 1311 } 1312 1313 /* 1314 * If plen does not account for pkt size 1315 */ 1316 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1317 abort_connection(ep, skb, GFP_KERNEL); 1318 return; 1319 } 1320 ep->plen = (u8) plen; 1321 1322 /* 1323 * If we don't have all the pdata yet, then bail. 1324 */ 1325 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1326 return; 1327 1328 /* 1329 * If we get here we have accumulated the entire mpa 1330 * start reply message including private data. 1331 */ 1332 ep->mpa_attr.initiator = 0; 1333 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1334 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1335 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1336 ep->mpa_attr.version = mpa->revision; 1337 if (mpa->revision == 1) 1338 ep->tried_with_mpa_v1 = 1; 1339 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1340 1341 if (mpa->revision == 2) { 1342 ep->mpa_attr.enhanced_rdma_conn = 1343 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1344 if (ep->mpa_attr.enhanced_rdma_conn) { 1345 mpa_v2_params = (struct mpa_v2_conn_params *) 1346 (ep->mpa_pkt + sizeof(*mpa)); 1347 ep->ird = ntohs(mpa_v2_params->ird) & 1348 MPA_V2_IRD_ORD_MASK; 1349 ep->ord = ntohs(mpa_v2_params->ord) & 1350 MPA_V2_IRD_ORD_MASK; 1351 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1352 if (peer2peer) { 1353 if (ntohs(mpa_v2_params->ord) & 1354 MPA_V2_RDMA_WRITE_RTR) 1355 ep->mpa_attr.p2p_type = 1356 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1357 else if (ntohs(mpa_v2_params->ord) & 1358 MPA_V2_RDMA_READ_RTR) 1359 ep->mpa_attr.p2p_type = 1360 FW_RI_INIT_P2PTYPE_READ_REQ; 1361 } 1362 } 1363 } else if (mpa->revision == 1) 1364 if (peer2peer) 1365 ep->mpa_attr.p2p_type = p2p_type; 1366 1367 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1368 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1369 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1370 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1371 ep->mpa_attr.p2p_type); 1372 1373 state_set(&ep->com, MPA_REQ_RCVD); 1374 1375 /* drive upcall */ 1376 connect_request_upcall(ep); 1377 return; 1378 } 1379 1380 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1381 { 1382 struct c4iw_ep *ep; 1383 struct cpl_rx_data *hdr = cplhdr(skb); 1384 unsigned int dlen = ntohs(hdr->len); 1385 unsigned int tid = GET_TID(hdr); 1386 struct tid_info *t = dev->rdev.lldi.tids; 1387 __u8 status = hdr->status; 1388 1389 ep = lookup_tid(t, tid); 1390 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1391 skb_pull(skb, sizeof(*hdr)); 1392 skb_trim(skb, dlen); 1393 1394 ep->rcv_seq += dlen; 1395 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen)); 1396 1397 /* update RX credits */ 1398 update_rx_credits(ep, dlen); 1399 1400 switch (state_read(&ep->com)) { 1401 case MPA_REQ_SENT: 1402 process_mpa_reply(ep, skb); 1403 break; 1404 case MPA_REQ_WAIT: 1405 process_mpa_request(ep, skb); 1406 break; 1407 case MPA_REP_SENT: 1408 break; 1409 default: 1410 pr_err("%s Unexpected streaming data." \ 1411 " ep %p state %d tid %u status %d\n", 1412 __func__, ep, state_read(&ep->com), ep->hwtid, status); 1413 1414 /* 1415 * The ep will timeout and inform the ULP of the failure. 1416 * See ep_timeout(). 1417 */ 1418 break; 1419 } 1420 return 0; 1421 } 1422 1423 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1424 { 1425 struct c4iw_ep *ep; 1426 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1427 int release = 0; 1428 unsigned int tid = GET_TID(rpl); 1429 struct tid_info *t = dev->rdev.lldi.tids; 1430 1431 ep = lookup_tid(t, tid); 1432 if (!ep) { 1433 printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n"); 1434 return 0; 1435 } 1436 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1437 mutex_lock(&ep->com.mutex); 1438 switch (ep->com.state) { 1439 case ABORTING: 1440 __state_set(&ep->com, DEAD); 1441 release = 1; 1442 break; 1443 default: 1444 printk(KERN_ERR "%s ep %p state %d\n", 1445 __func__, ep, ep->com.state); 1446 break; 1447 } 1448 mutex_unlock(&ep->com.mutex); 1449 1450 if (release) 1451 release_ep_resources(ep); 1452 return 0; 1453 } 1454 1455 static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid) 1456 { 1457 struct sk_buff *skb; 1458 struct fw_ofld_connection_wr *req; 1459 unsigned int mtu_idx; 1460 int wscale; 1461 1462 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1463 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1464 memset(req, 0, sizeof(*req)); 1465 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR)); 1466 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1467 req->le.filter = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, 1468 ep->l2t)); 1469 req->le.lport = ep->com.local_addr.sin_port; 1470 req->le.pport = ep->com.remote_addr.sin_port; 1471 req->le.u.ipv4.lip = ep->com.local_addr.sin_addr.s_addr; 1472 req->le.u.ipv4.pip = ep->com.remote_addr.sin_addr.s_addr; 1473 req->tcb.t_state_to_astid = 1474 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1475 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1476 req->tcb.cplrxdataack_cplpassacceptrpl = 1477 htons(F_FW_OFLD_CONNECTION_WR_CPLRXDATAACK); 1478 req->tcb.tx_max = jiffies; 1479 req->tcb.rcv_adv = htons(1); 1480 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1481 wscale = compute_wscale(rcv_win); 1482 req->tcb.opt0 = TCAM_BYPASS(1) | 1483 (nocong ? NO_CONG(1) : 0) | 1484 KEEP_ALIVE(1) | 1485 DELACK(1) | 1486 WND_SCALE(wscale) | 1487 MSS_IDX(mtu_idx) | 1488 L2T_IDX(ep->l2t->idx) | 1489 TX_CHAN(ep->tx_chan) | 1490 SMAC_SEL(ep->smac_idx) | 1491 DSCP(ep->tos) | 1492 ULP_MODE(ULP_MODE_TCPDDP) | 1493 RCV_BUFSIZ(rcv_win >> 10); 1494 req->tcb.opt2 = PACE(1) | 1495 TX_QUEUE(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) | 1496 RX_CHANNEL(0) | 1497 CCTRL_ECN(enable_ecn) | 1498 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1499 if (enable_tcp_timestamps) 1500 req->tcb.opt2 |= TSTAMPS_EN(1); 1501 if (enable_tcp_sack) 1502 req->tcb.opt2 |= SACK_EN(1); 1503 if (wscale && enable_tcp_window_scaling) 1504 req->tcb.opt2 |= WND_SCALE_EN(1); 1505 req->tcb.opt0 = cpu_to_be64(req->tcb.opt0); 1506 req->tcb.opt2 = cpu_to_be32(req->tcb.opt2); 1507 set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx); 1508 set_bit(ACT_OFLD_CONN, &ep->com.history); 1509 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1510 } 1511 1512 /* 1513 * Return whether a failed active open has allocated a TID 1514 */ 1515 static inline int act_open_has_tid(int status) 1516 { 1517 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1518 status != CPL_ERR_ARP_MISS; 1519 } 1520 1521 #define ACT_OPEN_RETRY_COUNT 2 1522 1523 static int c4iw_reconnect(struct c4iw_ep *ep) 1524 { 1525 int err = 0; 1526 struct rtable *rt; 1527 struct port_info *pi; 1528 struct net_device *pdev; 1529 int step; 1530 struct neighbour *neigh; 1531 1532 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1533 init_timer(&ep->timer); 1534 1535 /* 1536 * Allocate an active TID to initiate a TCP connection. 1537 */ 1538 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1539 if (ep->atid == -1) { 1540 pr_err("%s - cannot alloc atid.\n", __func__); 1541 err = -ENOMEM; 1542 goto fail2; 1543 } 1544 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1545 1546 /* find a route */ 1547 rt = find_route(ep->com.dev, 1548 ep->com.cm_id->local_addr.sin_addr.s_addr, 1549 ep->com.cm_id->remote_addr.sin_addr.s_addr, 1550 ep->com.cm_id->local_addr.sin_port, 1551 ep->com.cm_id->remote_addr.sin_port, 0); 1552 if (!rt) { 1553 pr_err("%s - cannot find route.\n", __func__); 1554 err = -EHOSTUNREACH; 1555 goto fail3; 1556 } 1557 ep->dst = &rt->dst; 1558 1559 neigh = dst_neigh_lookup(ep->dst, 1560 &ep->com.cm_id->remote_addr.sin_addr.s_addr); 1561 /* get a l2t entry */ 1562 if (neigh->dev->flags & IFF_LOOPBACK) { 1563 PDBG("%s LOOPBACK\n", __func__); 1564 pdev = ip_dev_find(&init_net, 1565 ep->com.cm_id->remote_addr.sin_addr.s_addr); 1566 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t, 1567 neigh, pdev, 0); 1568 pi = (struct port_info *)netdev_priv(pdev); 1569 ep->mtu = pdev->mtu; 1570 ep->tx_chan = cxgb4_port_chan(pdev); 1571 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1572 dev_put(pdev); 1573 } else { 1574 ep->l2t = cxgb4_l2t_get(ep->com.dev->rdev.lldi.l2t, 1575 neigh, neigh->dev, 0); 1576 pi = (struct port_info *)netdev_priv(neigh->dev); 1577 ep->mtu = dst_mtu(ep->dst); 1578 ep->tx_chan = cxgb4_port_chan(neigh->dev); 1579 ep->smac_idx = (cxgb4_port_viid(neigh->dev) & 1580 0x7F) << 1; 1581 } 1582 1583 step = ep->com.dev->rdev.lldi.ntxq / ep->com.dev->rdev.lldi.nchan; 1584 ep->txq_idx = pi->port_id * step; 1585 ep->ctrlq_idx = pi->port_id; 1586 step = ep->com.dev->rdev.lldi.nrxq / ep->com.dev->rdev.lldi.nchan; 1587 ep->rss_qid = ep->com.dev->rdev.lldi.rxq_ids[pi->port_id * step]; 1588 1589 if (!ep->l2t) { 1590 pr_err("%s - cannot alloc l2e.\n", __func__); 1591 err = -ENOMEM; 1592 goto fail4; 1593 } 1594 1595 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1596 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1597 ep->l2t->idx); 1598 1599 state_set(&ep->com, CONNECTING); 1600 ep->tos = 0; 1601 1602 /* send connect request to rnic */ 1603 err = send_connect(ep); 1604 if (!err) 1605 goto out; 1606 1607 cxgb4_l2t_release(ep->l2t); 1608 fail4: 1609 dst_release(ep->dst); 1610 fail3: 1611 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 1612 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1613 fail2: 1614 /* 1615 * remember to send notification to upper layer. 1616 * We are in here so the upper layer is not aware that this is 1617 * re-connect attempt and so, upper layer is still waiting for 1618 * response of 1st connect request. 1619 */ 1620 connect_reply_upcall(ep, -ECONNRESET); 1621 c4iw_put_ep(&ep->com); 1622 out: 1623 return err; 1624 } 1625 1626 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1627 { 1628 struct c4iw_ep *ep; 1629 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1630 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1631 ntohl(rpl->atid_status))); 1632 struct tid_info *t = dev->rdev.lldi.tids; 1633 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1634 1635 ep = lookup_atid(t, atid); 1636 1637 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1638 status, status2errno(status)); 1639 1640 if (status == CPL_ERR_RTX_NEG_ADVICE) { 1641 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1642 atid); 1643 return 0; 1644 } 1645 1646 set_bit(ACT_OPEN_RPL, &ep->com.history); 1647 1648 /* 1649 * Log interesting failures. 1650 */ 1651 switch (status) { 1652 case CPL_ERR_CONN_RESET: 1653 case CPL_ERR_CONN_TIMEDOUT: 1654 break; 1655 case CPL_ERR_TCAM_FULL: 1656 if (dev->rdev.lldi.enable_fw_ofld_conn) { 1657 mutex_lock(&dev->rdev.stats.lock); 1658 dev->rdev.stats.tcam_full++; 1659 mutex_unlock(&dev->rdev.stats.lock); 1660 send_fw_act_open_req(ep, 1661 GET_TID_TID(GET_AOPEN_ATID( 1662 ntohl(rpl->atid_status)))); 1663 return 0; 1664 } 1665 break; 1666 case CPL_ERR_CONN_EXIST: 1667 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 1668 set_bit(ACT_RETRY_INUSE, &ep->com.history); 1669 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, 1670 atid); 1671 cxgb4_free_atid(t, atid); 1672 dst_release(ep->dst); 1673 cxgb4_l2t_release(ep->l2t); 1674 c4iw_reconnect(ep); 1675 return 0; 1676 } 1677 break; 1678 default: 1679 printk(KERN_INFO MOD "Active open failure - " 1680 "atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1681 atid, status, status2errno(status), 1682 &ep->com.local_addr.sin_addr.s_addr, 1683 ntohs(ep->com.local_addr.sin_port), 1684 &ep->com.remote_addr.sin_addr.s_addr, 1685 ntohs(ep->com.remote_addr.sin_port)); 1686 break; 1687 } 1688 1689 connect_reply_upcall(ep, status2errno(status)); 1690 state_set(&ep->com, DEAD); 1691 1692 if (status && act_open_has_tid(status)) 1693 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1694 1695 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 1696 cxgb4_free_atid(t, atid); 1697 dst_release(ep->dst); 1698 cxgb4_l2t_release(ep->l2t); 1699 c4iw_put_ep(&ep->com); 1700 1701 return 0; 1702 } 1703 1704 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1705 { 1706 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1707 struct tid_info *t = dev->rdev.lldi.tids; 1708 unsigned int stid = GET_TID(rpl); 1709 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1710 1711 if (!ep) { 1712 PDBG("%s stid %d lookup failure!\n", __func__, stid); 1713 goto out; 1714 } 1715 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1716 rpl->status, status2errno(rpl->status)); 1717 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1718 1719 out: 1720 return 0; 1721 } 1722 1723 static int listen_stop(struct c4iw_listen_ep *ep) 1724 { 1725 struct sk_buff *skb; 1726 struct cpl_close_listsvr_req *req; 1727 1728 PDBG("%s ep %p\n", __func__, ep); 1729 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1730 if (!skb) { 1731 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 1732 return -ENOMEM; 1733 } 1734 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req)); 1735 INIT_TP_WR(req, 0); 1736 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 1737 ep->stid)); 1738 req->reply_ctrl = cpu_to_be16( 1739 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0])); 1740 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 1741 return c4iw_ofld_send(&ep->com.dev->rdev, skb); 1742 } 1743 1744 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1745 { 1746 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1747 struct tid_info *t = dev->rdev.lldi.tids; 1748 unsigned int stid = GET_TID(rpl); 1749 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1750 1751 PDBG("%s ep %p\n", __func__, ep); 1752 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1753 return 0; 1754 } 1755 1756 static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, 1757 struct cpl_pass_accept_req *req) 1758 { 1759 struct cpl_pass_accept_rpl *rpl; 1760 unsigned int mtu_idx; 1761 u64 opt0; 1762 u32 opt2; 1763 int wscale; 1764 1765 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1766 BUG_ON(skb_cloned(skb)); 1767 skb_trim(skb, sizeof(*rpl)); 1768 skb_get(skb); 1769 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1770 wscale = compute_wscale(rcv_win); 1771 opt0 = (nocong ? NO_CONG(1) : 0) | 1772 KEEP_ALIVE(1) | 1773 DELACK(1) | 1774 WND_SCALE(wscale) | 1775 MSS_IDX(mtu_idx) | 1776 L2T_IDX(ep->l2t->idx) | 1777 TX_CHAN(ep->tx_chan) | 1778 SMAC_SEL(ep->smac_idx) | 1779 DSCP(ep->tos >> 2) | 1780 ULP_MODE(ULP_MODE_TCPDDP) | 1781 RCV_BUFSIZ(rcv_win>>10); 1782 opt2 = RX_CHANNEL(0) | 1783 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1784 1785 if (enable_tcp_timestamps && req->tcpopt.tstamp) 1786 opt2 |= TSTAMPS_EN(1); 1787 if (enable_tcp_sack && req->tcpopt.sack) 1788 opt2 |= SACK_EN(1); 1789 if (wscale && enable_tcp_window_scaling) 1790 opt2 |= WND_SCALE_EN(1); 1791 if (enable_ecn) { 1792 const struct tcphdr *tcph; 1793 u32 hlen = ntohl(req->hdr_len); 1794 1795 tcph = (const void *)(req + 1) + G_ETH_HDR_LEN(hlen) + 1796 G_IP_HDR_LEN(hlen); 1797 if (tcph->ece && tcph->cwr) 1798 opt2 |= CCTRL_ECN(1); 1799 } 1800 1801 rpl = cplhdr(skb); 1802 INIT_TP_WR(rpl, ep->hwtid); 1803 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1804 ep->hwtid)); 1805 rpl->opt0 = cpu_to_be64(opt0); 1806 rpl->opt2 = cpu_to_be32(opt2); 1807 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 1808 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1809 1810 return; 1811 } 1812 1813 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip, 1814 struct sk_buff *skb) 1815 { 1816 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid, 1817 peer_ip); 1818 BUG_ON(skb_cloned(skb)); 1819 skb_trim(skb, sizeof(struct cpl_tid_release)); 1820 skb_get(skb); 1821 release_tid(&dev->rdev, hwtid, skb); 1822 return; 1823 } 1824 1825 static void get_4tuple(struct cpl_pass_accept_req *req, 1826 __be32 *local_ip, __be32 *peer_ip, 1827 __be16 *local_port, __be16 *peer_port) 1828 { 1829 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 1830 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 1831 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 1832 struct tcphdr *tcp = (struct tcphdr *) 1833 ((u8 *)(req + 1) + eth_len + ip_len); 1834 1835 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 1836 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 1837 ntohs(tcp->dest)); 1838 1839 *peer_ip = ip->saddr; 1840 *local_ip = ip->daddr; 1841 *peer_port = tcp->source; 1842 *local_port = tcp->dest; 1843 1844 return; 1845 } 1846 1847 static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst, 1848 struct c4iw_dev *cdev, bool clear_mpa_v1) 1849 { 1850 struct neighbour *n; 1851 int err, step; 1852 1853 n = dst_neigh_lookup(dst, &peer_ip); 1854 if (!n) 1855 return -ENODEV; 1856 1857 rcu_read_lock(); 1858 err = -ENOMEM; 1859 if (n->dev->flags & IFF_LOOPBACK) { 1860 struct net_device *pdev; 1861 1862 pdev = ip_dev_find(&init_net, peer_ip); 1863 if (!pdev) { 1864 err = -ENODEV; 1865 goto out; 1866 } 1867 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1868 n, pdev, 0); 1869 if (!ep->l2t) 1870 goto out; 1871 ep->mtu = pdev->mtu; 1872 ep->tx_chan = cxgb4_port_chan(pdev); 1873 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1874 step = cdev->rdev.lldi.ntxq / 1875 cdev->rdev.lldi.nchan; 1876 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1877 step = cdev->rdev.lldi.nrxq / 1878 cdev->rdev.lldi.nchan; 1879 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1880 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1881 cxgb4_port_idx(pdev) * step]; 1882 dev_put(pdev); 1883 } else { 1884 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1885 n, n->dev, 0); 1886 if (!ep->l2t) 1887 goto out; 1888 ep->mtu = dst_mtu(dst); 1889 ep->tx_chan = cxgb4_port_chan(n->dev); 1890 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1891 step = cdev->rdev.lldi.ntxq / 1892 cdev->rdev.lldi.nchan; 1893 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1894 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1895 step = cdev->rdev.lldi.nrxq / 1896 cdev->rdev.lldi.nchan; 1897 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1898 cxgb4_port_idx(n->dev) * step]; 1899 1900 if (clear_mpa_v1) { 1901 ep->retry_with_mpa_v1 = 0; 1902 ep->tried_with_mpa_v1 = 0; 1903 } 1904 } 1905 err = 0; 1906 out: 1907 rcu_read_unlock(); 1908 1909 neigh_release(n); 1910 1911 return err; 1912 } 1913 1914 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 1915 { 1916 struct c4iw_ep *child_ep = NULL, *parent_ep; 1917 struct cpl_pass_accept_req *req = cplhdr(skb); 1918 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 1919 struct tid_info *t = dev->rdev.lldi.tids; 1920 unsigned int hwtid = GET_TID(req); 1921 struct dst_entry *dst; 1922 struct rtable *rt; 1923 __be32 local_ip, peer_ip = 0; 1924 __be16 local_port, peer_port; 1925 int err; 1926 u16 peer_mss = ntohs(req->tcpopt.mss); 1927 1928 parent_ep = lookup_stid(t, stid); 1929 if (!parent_ep) { 1930 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 1931 goto reject; 1932 } 1933 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port); 1934 1935 PDBG("%s parent ep %p hwtid %u laddr 0x%x raddr 0x%x lport %d " \ 1936 "rport %d peer_mss %d\n", __func__, parent_ep, hwtid, 1937 ntohl(local_ip), ntohl(peer_ip), ntohs(local_port), 1938 ntohs(peer_port), peer_mss); 1939 1940 if (state_read(&parent_ep->com) != LISTEN) { 1941 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 1942 __func__); 1943 goto reject; 1944 } 1945 1946 /* Find output route */ 1947 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port, 1948 GET_POPEN_TOS(ntohl(req->tos_stid))); 1949 if (!rt) { 1950 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 1951 __func__); 1952 goto reject; 1953 } 1954 dst = &rt->dst; 1955 1956 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 1957 if (!child_ep) { 1958 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 1959 __func__); 1960 dst_release(dst); 1961 goto reject; 1962 } 1963 1964 err = import_ep(child_ep, peer_ip, dst, dev, false); 1965 if (err) { 1966 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 1967 __func__); 1968 dst_release(dst); 1969 kfree(child_ep); 1970 goto reject; 1971 } 1972 1973 if (peer_mss && child_ep->mtu > (peer_mss + 40)) 1974 child_ep->mtu = peer_mss + 40; 1975 1976 state_set(&child_ep->com, CONNECTING); 1977 child_ep->com.dev = dev; 1978 child_ep->com.cm_id = NULL; 1979 child_ep->com.local_addr.sin_family = PF_INET; 1980 child_ep->com.local_addr.sin_port = local_port; 1981 child_ep->com.local_addr.sin_addr.s_addr = local_ip; 1982 child_ep->com.remote_addr.sin_family = PF_INET; 1983 child_ep->com.remote_addr.sin_port = peer_port; 1984 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip; 1985 c4iw_get_ep(&parent_ep->com); 1986 child_ep->parent_ep = parent_ep; 1987 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 1988 child_ep->dst = dst; 1989 child_ep->hwtid = hwtid; 1990 1991 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 1992 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 1993 1994 init_timer(&child_ep->timer); 1995 cxgb4_insert_tid(t, child_ep, hwtid); 1996 accept_cr(child_ep, peer_ip, skb, req); 1997 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 1998 goto out; 1999 reject: 2000 reject_cr(dev, hwtid, peer_ip, skb); 2001 out: 2002 return 0; 2003 } 2004 2005 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 2006 { 2007 struct c4iw_ep *ep; 2008 struct cpl_pass_establish *req = cplhdr(skb); 2009 struct tid_info *t = dev->rdev.lldi.tids; 2010 unsigned int tid = GET_TID(req); 2011 2012 ep = lookup_tid(t, tid); 2013 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2014 ep->snd_seq = be32_to_cpu(req->snd_isn); 2015 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 2016 2017 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__, ep, tid, 2018 ntohs(req->tcp_opt)); 2019 2020 set_emss(ep, ntohs(req->tcp_opt)); 2021 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 2022 2023 dst_confirm(ep->dst); 2024 state_set(&ep->com, MPA_REQ_WAIT); 2025 start_ep_timer(ep); 2026 send_flowc(ep, skb); 2027 set_bit(PASS_ESTAB, &ep->com.history); 2028 2029 return 0; 2030 } 2031 2032 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 2033 { 2034 struct cpl_peer_close *hdr = cplhdr(skb); 2035 struct c4iw_ep *ep; 2036 struct c4iw_qp_attributes attrs; 2037 int disconnect = 1; 2038 int release = 0; 2039 struct tid_info *t = dev->rdev.lldi.tids; 2040 unsigned int tid = GET_TID(hdr); 2041 int ret; 2042 2043 ep = lookup_tid(t, tid); 2044 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2045 dst_confirm(ep->dst); 2046 2047 set_bit(PEER_CLOSE, &ep->com.history); 2048 mutex_lock(&ep->com.mutex); 2049 switch (ep->com.state) { 2050 case MPA_REQ_WAIT: 2051 __state_set(&ep->com, CLOSING); 2052 break; 2053 case MPA_REQ_SENT: 2054 __state_set(&ep->com, CLOSING); 2055 connect_reply_upcall(ep, -ECONNRESET); 2056 break; 2057 case MPA_REQ_RCVD: 2058 2059 /* 2060 * We're gonna mark this puppy DEAD, but keep 2061 * the reference on it until the ULP accepts or 2062 * rejects the CR. Also wake up anyone waiting 2063 * in rdma connection migration (see c4iw_accept_cr()). 2064 */ 2065 __state_set(&ep->com, CLOSING); 2066 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2067 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2068 break; 2069 case MPA_REP_SENT: 2070 __state_set(&ep->com, CLOSING); 2071 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2072 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2073 break; 2074 case FPDU_MODE: 2075 start_ep_timer(ep); 2076 __state_set(&ep->com, CLOSING); 2077 attrs.next_state = C4IW_QP_STATE_CLOSING; 2078 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2079 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2080 if (ret != -ECONNRESET) { 2081 peer_close_upcall(ep); 2082 disconnect = 1; 2083 } 2084 break; 2085 case ABORTING: 2086 disconnect = 0; 2087 break; 2088 case CLOSING: 2089 __state_set(&ep->com, MORIBUND); 2090 disconnect = 0; 2091 break; 2092 case MORIBUND: 2093 stop_ep_timer(ep); 2094 if (ep->com.cm_id && ep->com.qp) { 2095 attrs.next_state = C4IW_QP_STATE_IDLE; 2096 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2097 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2098 } 2099 close_complete_upcall(ep); 2100 __state_set(&ep->com, DEAD); 2101 release = 1; 2102 disconnect = 0; 2103 break; 2104 case DEAD: 2105 disconnect = 0; 2106 break; 2107 default: 2108 BUG_ON(1); 2109 } 2110 mutex_unlock(&ep->com.mutex); 2111 if (disconnect) 2112 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2113 if (release) 2114 release_ep_resources(ep); 2115 return 0; 2116 } 2117 2118 /* 2119 * Returns whether an ABORT_REQ_RSS message is a negative advice. 2120 */ 2121 static int is_neg_adv_abort(unsigned int status) 2122 { 2123 return status == CPL_ERR_RTX_NEG_ADVICE || 2124 status == CPL_ERR_PERSIST_NEG_ADVICE; 2125 } 2126 2127 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 2128 { 2129 struct cpl_abort_req_rss *req = cplhdr(skb); 2130 struct c4iw_ep *ep; 2131 struct cpl_abort_rpl *rpl; 2132 struct sk_buff *rpl_skb; 2133 struct c4iw_qp_attributes attrs; 2134 int ret; 2135 int release = 0; 2136 struct tid_info *t = dev->rdev.lldi.tids; 2137 unsigned int tid = GET_TID(req); 2138 2139 ep = lookup_tid(t, tid); 2140 if (is_neg_adv_abort(req->status)) { 2141 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2142 ep->hwtid); 2143 return 0; 2144 } 2145 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2146 ep->com.state); 2147 set_bit(PEER_ABORT, &ep->com.history); 2148 2149 /* 2150 * Wake up any threads in rdma_init() or rdma_fini(). 2151 * However, this is not needed if com state is just 2152 * MPA_REQ_SENT 2153 */ 2154 if (ep->com.state != MPA_REQ_SENT) 2155 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2156 2157 mutex_lock(&ep->com.mutex); 2158 switch (ep->com.state) { 2159 case CONNECTING: 2160 break; 2161 case MPA_REQ_WAIT: 2162 stop_ep_timer(ep); 2163 break; 2164 case MPA_REQ_SENT: 2165 stop_ep_timer(ep); 2166 if (mpa_rev == 2 && ep->tried_with_mpa_v1) 2167 connect_reply_upcall(ep, -ECONNRESET); 2168 else { 2169 /* 2170 * we just don't send notification upwards because we 2171 * want to retry with mpa_v1 without upper layers even 2172 * knowing it. 2173 * 2174 * do some housekeeping so as to re-initiate the 2175 * connection 2176 */ 2177 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 2178 mpa_rev); 2179 ep->retry_with_mpa_v1 = 1; 2180 } 2181 break; 2182 case MPA_REP_SENT: 2183 break; 2184 case MPA_REQ_RCVD: 2185 break; 2186 case MORIBUND: 2187 case CLOSING: 2188 stop_ep_timer(ep); 2189 /*FALLTHROUGH*/ 2190 case FPDU_MODE: 2191 if (ep->com.cm_id && ep->com.qp) { 2192 attrs.next_state = C4IW_QP_STATE_ERROR; 2193 ret = c4iw_modify_qp(ep->com.qp->rhp, 2194 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2195 &attrs, 1); 2196 if (ret) 2197 printk(KERN_ERR MOD 2198 "%s - qp <- error failed!\n", 2199 __func__); 2200 } 2201 peer_abort_upcall(ep); 2202 break; 2203 case ABORTING: 2204 break; 2205 case DEAD: 2206 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 2207 mutex_unlock(&ep->com.mutex); 2208 return 0; 2209 default: 2210 BUG_ON(1); 2211 break; 2212 } 2213 dst_confirm(ep->dst); 2214 if (ep->com.state != ABORTING) { 2215 __state_set(&ep->com, DEAD); 2216 /* we don't release if we want to retry with mpa_v1 */ 2217 if (!ep->retry_with_mpa_v1) 2218 release = 1; 2219 } 2220 mutex_unlock(&ep->com.mutex); 2221 2222 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 2223 if (!rpl_skb) { 2224 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 2225 __func__); 2226 release = 1; 2227 goto out; 2228 } 2229 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 2230 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 2231 INIT_TP_WR(rpl, ep->hwtid); 2232 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 2233 rpl->cmd = CPL_ABORT_NO_RST; 2234 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 2235 out: 2236 if (release) 2237 release_ep_resources(ep); 2238 2239 /* retry with mpa-v1 */ 2240 if (ep && ep->retry_with_mpa_v1) { 2241 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 2242 dst_release(ep->dst); 2243 cxgb4_l2t_release(ep->l2t); 2244 c4iw_reconnect(ep); 2245 } 2246 2247 return 0; 2248 } 2249 2250 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2251 { 2252 struct c4iw_ep *ep; 2253 struct c4iw_qp_attributes attrs; 2254 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2255 int release = 0; 2256 struct tid_info *t = dev->rdev.lldi.tids; 2257 unsigned int tid = GET_TID(rpl); 2258 2259 ep = lookup_tid(t, tid); 2260 2261 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2262 BUG_ON(!ep); 2263 2264 /* The cm_id may be null if we failed to connect */ 2265 mutex_lock(&ep->com.mutex); 2266 switch (ep->com.state) { 2267 case CLOSING: 2268 __state_set(&ep->com, MORIBUND); 2269 break; 2270 case MORIBUND: 2271 stop_ep_timer(ep); 2272 if ((ep->com.cm_id) && (ep->com.qp)) { 2273 attrs.next_state = C4IW_QP_STATE_IDLE; 2274 c4iw_modify_qp(ep->com.qp->rhp, 2275 ep->com.qp, 2276 C4IW_QP_ATTR_NEXT_STATE, 2277 &attrs, 1); 2278 } 2279 close_complete_upcall(ep); 2280 __state_set(&ep->com, DEAD); 2281 release = 1; 2282 break; 2283 case ABORTING: 2284 case DEAD: 2285 break; 2286 default: 2287 BUG_ON(1); 2288 break; 2289 } 2290 mutex_unlock(&ep->com.mutex); 2291 if (release) 2292 release_ep_resources(ep); 2293 return 0; 2294 } 2295 2296 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2297 { 2298 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2299 struct tid_info *t = dev->rdev.lldi.tids; 2300 unsigned int tid = GET_TID(rpl); 2301 struct c4iw_ep *ep; 2302 struct c4iw_qp_attributes attrs; 2303 2304 ep = lookup_tid(t, tid); 2305 BUG_ON(!ep); 2306 2307 if (ep && ep->com.qp) { 2308 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2309 ep->com.qp->wq.sq.qid); 2310 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2311 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2312 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2313 } else 2314 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2315 2316 return 0; 2317 } 2318 2319 /* 2320 * Upcall from the adapter indicating data has been transmitted. 2321 * For us its just the single MPA request or reply. We can now free 2322 * the skb holding the mpa message. 2323 */ 2324 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2325 { 2326 struct c4iw_ep *ep; 2327 struct cpl_fw4_ack *hdr = cplhdr(skb); 2328 u8 credits = hdr->credits; 2329 unsigned int tid = GET_TID(hdr); 2330 struct tid_info *t = dev->rdev.lldi.tids; 2331 2332 2333 ep = lookup_tid(t, tid); 2334 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2335 if (credits == 0) { 2336 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2337 __func__, ep, ep->hwtid, state_read(&ep->com)); 2338 return 0; 2339 } 2340 2341 dst_confirm(ep->dst); 2342 if (ep->mpa_skb) { 2343 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2344 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2345 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2346 kfree_skb(ep->mpa_skb); 2347 ep->mpa_skb = NULL; 2348 } 2349 return 0; 2350 } 2351 2352 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2353 { 2354 int err; 2355 struct c4iw_ep *ep = to_ep(cm_id); 2356 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2357 2358 if (state_read(&ep->com) == DEAD) { 2359 c4iw_put_ep(&ep->com); 2360 return -ECONNRESET; 2361 } 2362 set_bit(ULP_REJECT, &ep->com.history); 2363 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2364 if (mpa_rev == 0) 2365 abort_connection(ep, NULL, GFP_KERNEL); 2366 else { 2367 err = send_mpa_reject(ep, pdata, pdata_len); 2368 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2369 } 2370 c4iw_put_ep(&ep->com); 2371 return 0; 2372 } 2373 2374 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2375 { 2376 int err; 2377 struct c4iw_qp_attributes attrs; 2378 enum c4iw_qp_attr_mask mask; 2379 struct c4iw_ep *ep = to_ep(cm_id); 2380 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2381 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2382 2383 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2384 if (state_read(&ep->com) == DEAD) { 2385 err = -ECONNRESET; 2386 goto err; 2387 } 2388 2389 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2390 BUG_ON(!qp); 2391 2392 set_bit(ULP_ACCEPT, &ep->com.history); 2393 if ((conn_param->ord > c4iw_max_read_depth) || 2394 (conn_param->ird > c4iw_max_read_depth)) { 2395 abort_connection(ep, NULL, GFP_KERNEL); 2396 err = -EINVAL; 2397 goto err; 2398 } 2399 2400 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2401 if (conn_param->ord > ep->ird) { 2402 ep->ird = conn_param->ird; 2403 ep->ord = conn_param->ord; 2404 send_mpa_reject(ep, conn_param->private_data, 2405 conn_param->private_data_len); 2406 abort_connection(ep, NULL, GFP_KERNEL); 2407 err = -ENOMEM; 2408 goto err; 2409 } 2410 if (conn_param->ird > ep->ord) { 2411 if (!ep->ord) 2412 conn_param->ird = 1; 2413 else { 2414 abort_connection(ep, NULL, GFP_KERNEL); 2415 err = -ENOMEM; 2416 goto err; 2417 } 2418 } 2419 2420 } 2421 ep->ird = conn_param->ird; 2422 ep->ord = conn_param->ord; 2423 2424 if (ep->mpa_attr.version != 2) 2425 if (peer2peer && ep->ird == 0) 2426 ep->ird = 1; 2427 2428 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2429 2430 cm_id->add_ref(cm_id); 2431 ep->com.cm_id = cm_id; 2432 ep->com.qp = qp; 2433 2434 /* bind QP to EP and move to RTS */ 2435 attrs.mpa_attr = ep->mpa_attr; 2436 attrs.max_ird = ep->ird; 2437 attrs.max_ord = ep->ord; 2438 attrs.llp_stream_handle = ep; 2439 attrs.next_state = C4IW_QP_STATE_RTS; 2440 2441 /* bind QP and TID with INIT_WR */ 2442 mask = C4IW_QP_ATTR_NEXT_STATE | 2443 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2444 C4IW_QP_ATTR_MPA_ATTR | 2445 C4IW_QP_ATTR_MAX_IRD | 2446 C4IW_QP_ATTR_MAX_ORD; 2447 2448 err = c4iw_modify_qp(ep->com.qp->rhp, 2449 ep->com.qp, mask, &attrs, 1); 2450 if (err) 2451 goto err1; 2452 err = send_mpa_reply(ep, conn_param->private_data, 2453 conn_param->private_data_len); 2454 if (err) 2455 goto err1; 2456 2457 state_set(&ep->com, FPDU_MODE); 2458 established_upcall(ep); 2459 c4iw_put_ep(&ep->com); 2460 return 0; 2461 err1: 2462 ep->com.cm_id = NULL; 2463 ep->com.qp = NULL; 2464 cm_id->rem_ref(cm_id); 2465 err: 2466 c4iw_put_ep(&ep->com); 2467 return err; 2468 } 2469 2470 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2471 { 2472 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2473 struct c4iw_ep *ep; 2474 struct rtable *rt; 2475 int err = 0; 2476 2477 if ((conn_param->ord > c4iw_max_read_depth) || 2478 (conn_param->ird > c4iw_max_read_depth)) { 2479 err = -EINVAL; 2480 goto out; 2481 } 2482 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2483 if (!ep) { 2484 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2485 err = -ENOMEM; 2486 goto out; 2487 } 2488 init_timer(&ep->timer); 2489 ep->plen = conn_param->private_data_len; 2490 if (ep->plen) 2491 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2492 conn_param->private_data, ep->plen); 2493 ep->ird = conn_param->ird; 2494 ep->ord = conn_param->ord; 2495 2496 if (peer2peer && ep->ord == 0) 2497 ep->ord = 1; 2498 2499 cm_id->add_ref(cm_id); 2500 ep->com.dev = dev; 2501 ep->com.cm_id = cm_id; 2502 ep->com.qp = get_qhp(dev, conn_param->qpn); 2503 BUG_ON(!ep->com.qp); 2504 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2505 ep->com.qp, cm_id); 2506 2507 /* 2508 * Allocate an active TID to initiate a TCP connection. 2509 */ 2510 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2511 if (ep->atid == -1) { 2512 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2513 err = -ENOMEM; 2514 goto fail2; 2515 } 2516 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2517 2518 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__, 2519 ntohl(cm_id->local_addr.sin_addr.s_addr), 2520 ntohs(cm_id->local_addr.sin_port), 2521 ntohl(cm_id->remote_addr.sin_addr.s_addr), 2522 ntohs(cm_id->remote_addr.sin_port)); 2523 2524 /* find a route */ 2525 rt = find_route(dev, 2526 cm_id->local_addr.sin_addr.s_addr, 2527 cm_id->remote_addr.sin_addr.s_addr, 2528 cm_id->local_addr.sin_port, 2529 cm_id->remote_addr.sin_port, 0); 2530 if (!rt) { 2531 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2532 err = -EHOSTUNREACH; 2533 goto fail3; 2534 } 2535 ep->dst = &rt->dst; 2536 2537 err = import_ep(ep, cm_id->remote_addr.sin_addr.s_addr, 2538 ep->dst, ep->com.dev, true); 2539 if (err) { 2540 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2541 goto fail4; 2542 } 2543 2544 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2545 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2546 ep->l2t->idx); 2547 2548 state_set(&ep->com, CONNECTING); 2549 ep->tos = 0; 2550 ep->com.local_addr = cm_id->local_addr; 2551 ep->com.remote_addr = cm_id->remote_addr; 2552 2553 /* send connect request to rnic */ 2554 err = send_connect(ep); 2555 if (!err) 2556 goto out; 2557 2558 cxgb4_l2t_release(ep->l2t); 2559 fail4: 2560 dst_release(ep->dst); 2561 fail3: 2562 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 2563 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2564 fail2: 2565 cm_id->rem_ref(cm_id); 2566 c4iw_put_ep(&ep->com); 2567 out: 2568 return err; 2569 } 2570 2571 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2572 { 2573 int err = 0; 2574 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2575 struct c4iw_listen_ep *ep; 2576 2577 might_sleep(); 2578 2579 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2580 if (!ep) { 2581 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2582 err = -ENOMEM; 2583 goto fail1; 2584 } 2585 PDBG("%s ep %p\n", __func__, ep); 2586 cm_id->add_ref(cm_id); 2587 ep->com.cm_id = cm_id; 2588 ep->com.dev = dev; 2589 ep->backlog = backlog; 2590 ep->com.local_addr = cm_id->local_addr; 2591 2592 /* 2593 * Allocate a server TID. 2594 */ 2595 if (dev->rdev.lldi.enable_fw_ofld_conn) 2596 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, PF_INET, ep); 2597 else 2598 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep); 2599 2600 if (ep->stid == -1) { 2601 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2602 err = -ENOMEM; 2603 goto fail2; 2604 } 2605 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2606 state_set(&ep->com, LISTEN); 2607 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2608 do { 2609 err = cxgb4_create_server_filter( 2610 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2611 ep->com.local_addr.sin_addr.s_addr, 2612 ep->com.local_addr.sin_port, 2613 0, 2614 ep->com.dev->rdev.lldi.rxq_ids[0], 2615 0, 2616 0); 2617 if (err == -EBUSY) { 2618 set_current_state(TASK_UNINTERRUPTIBLE); 2619 schedule_timeout(usecs_to_jiffies(100)); 2620 } 2621 } while (err == -EBUSY); 2622 } else { 2623 c4iw_init_wr_wait(&ep->com.wr_wait); 2624 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], 2625 ep->stid, ep->com.local_addr.sin_addr.s_addr, 2626 ep->com.local_addr.sin_port, 2627 0, 2628 ep->com.dev->rdev.lldi.rxq_ids[0]); 2629 if (!err) 2630 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2631 &ep->com.wr_wait, 2632 0, 0, __func__); 2633 } 2634 if (!err) { 2635 cm_id->provider_data = ep; 2636 goto out; 2637 } 2638 pr_err("%s cxgb4_create_server/filter failed err %d " \ 2639 "stid %d laddr %08x lport %d\n", \ 2640 __func__, err, ep->stid, 2641 ntohl(ep->com.local_addr.sin_addr.s_addr), 2642 ntohs(ep->com.local_addr.sin_port)); 2643 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2644 fail2: 2645 cm_id->rem_ref(cm_id); 2646 c4iw_put_ep(&ep->com); 2647 fail1: 2648 out: 2649 return err; 2650 } 2651 2652 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2653 { 2654 int err; 2655 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2656 2657 PDBG("%s ep %p\n", __func__, ep); 2658 2659 might_sleep(); 2660 state_set(&ep->com, DEAD); 2661 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn) { 2662 err = cxgb4_remove_server_filter( 2663 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2664 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2665 } else { 2666 c4iw_init_wr_wait(&ep->com.wr_wait); 2667 err = listen_stop(ep); 2668 if (err) 2669 goto done; 2670 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 2671 0, 0, __func__); 2672 } 2673 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 2674 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2675 done: 2676 cm_id->rem_ref(cm_id); 2677 c4iw_put_ep(&ep->com); 2678 return err; 2679 } 2680 2681 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2682 { 2683 int ret = 0; 2684 int close = 0; 2685 int fatal = 0; 2686 struct c4iw_rdev *rdev; 2687 2688 mutex_lock(&ep->com.mutex); 2689 2690 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 2691 states[ep->com.state], abrupt); 2692 2693 rdev = &ep->com.dev->rdev; 2694 if (c4iw_fatal_error(rdev)) { 2695 fatal = 1; 2696 close_complete_upcall(ep); 2697 ep->com.state = DEAD; 2698 } 2699 switch (ep->com.state) { 2700 case MPA_REQ_WAIT: 2701 case MPA_REQ_SENT: 2702 case MPA_REQ_RCVD: 2703 case MPA_REP_SENT: 2704 case FPDU_MODE: 2705 close = 1; 2706 if (abrupt) 2707 ep->com.state = ABORTING; 2708 else { 2709 ep->com.state = CLOSING; 2710 start_ep_timer(ep); 2711 } 2712 set_bit(CLOSE_SENT, &ep->com.flags); 2713 break; 2714 case CLOSING: 2715 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2716 close = 1; 2717 if (abrupt) { 2718 stop_ep_timer(ep); 2719 ep->com.state = ABORTING; 2720 } else 2721 ep->com.state = MORIBUND; 2722 } 2723 break; 2724 case MORIBUND: 2725 case ABORTING: 2726 case DEAD: 2727 PDBG("%s ignoring disconnect ep %p state %u\n", 2728 __func__, ep, ep->com.state); 2729 break; 2730 default: 2731 BUG(); 2732 break; 2733 } 2734 2735 if (close) { 2736 if (abrupt) { 2737 set_bit(EP_DISC_ABORT, &ep->com.history); 2738 close_complete_upcall(ep); 2739 ret = send_abort(ep, NULL, gfp); 2740 } else { 2741 set_bit(EP_DISC_CLOSE, &ep->com.history); 2742 ret = send_halfclose(ep, gfp); 2743 } 2744 if (ret) 2745 fatal = 1; 2746 } 2747 mutex_unlock(&ep->com.mutex); 2748 if (fatal) 2749 release_ep_resources(ep); 2750 return ret; 2751 } 2752 2753 static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 2754 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 2755 { 2756 struct c4iw_ep *ep; 2757 int atid = be32_to_cpu(req->tid); 2758 2759 ep = (struct c4iw_ep *)lookup_atid(dev->rdev.lldi.tids, req->tid); 2760 if (!ep) 2761 return; 2762 2763 switch (req->retval) { 2764 case FW_ENOMEM: 2765 set_bit(ACT_RETRY_NOMEM, &ep->com.history); 2766 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 2767 send_fw_act_open_req(ep, atid); 2768 return; 2769 } 2770 case FW_EADDRINUSE: 2771 set_bit(ACT_RETRY_INUSE, &ep->com.history); 2772 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 2773 send_fw_act_open_req(ep, atid); 2774 return; 2775 } 2776 break; 2777 default: 2778 pr_info("%s unexpected ofld conn wr retval %d\n", 2779 __func__, req->retval); 2780 break; 2781 } 2782 pr_err("active ofld_connect_wr failure %d atid %d\n", 2783 req->retval, atid); 2784 mutex_lock(&dev->rdev.stats.lock); 2785 dev->rdev.stats.act_ofld_conn_fails++; 2786 mutex_unlock(&dev->rdev.stats.lock); 2787 connect_reply_upcall(ep, status2errno(req->retval)); 2788 state_set(&ep->com, DEAD); 2789 remove_handle(dev, &dev->atid_idr, atid); 2790 cxgb4_free_atid(dev->rdev.lldi.tids, atid); 2791 dst_release(ep->dst); 2792 cxgb4_l2t_release(ep->l2t); 2793 c4iw_put_ep(&ep->com); 2794 } 2795 2796 static void passive_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 2797 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 2798 { 2799 struct sk_buff *rpl_skb; 2800 struct cpl_pass_accept_req *cpl; 2801 int ret; 2802 2803 rpl_skb = (struct sk_buff *)cpu_to_be64(req->cookie); 2804 BUG_ON(!rpl_skb); 2805 if (req->retval) { 2806 PDBG("%s passive open failure %d\n", __func__, req->retval); 2807 mutex_lock(&dev->rdev.stats.lock); 2808 dev->rdev.stats.pas_ofld_conn_fails++; 2809 mutex_unlock(&dev->rdev.stats.lock); 2810 kfree_skb(rpl_skb); 2811 } else { 2812 cpl = (struct cpl_pass_accept_req *)cplhdr(rpl_skb); 2813 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 2814 htonl(req->tid))); 2815 ret = pass_accept_req(dev, rpl_skb); 2816 if (!ret) 2817 kfree_skb(rpl_skb); 2818 } 2819 return; 2820 } 2821 2822 static int deferred_fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 2823 { 2824 struct cpl_fw6_msg *rpl = cplhdr(skb); 2825 struct cpl_fw6_msg_ofld_connection_wr_rpl *req; 2826 2827 switch (rpl->type) { 2828 case FW6_TYPE_CQE: 2829 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 2830 break; 2831 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 2832 req = (struct cpl_fw6_msg_ofld_connection_wr_rpl *)rpl->data; 2833 switch (req->t_state) { 2834 case TCP_SYN_SENT: 2835 active_ofld_conn_reply(dev, skb, req); 2836 break; 2837 case TCP_SYN_RECV: 2838 passive_ofld_conn_reply(dev, skb, req); 2839 break; 2840 default: 2841 pr_err("%s unexpected ofld conn wr state %d\n", 2842 __func__, req->t_state); 2843 break; 2844 } 2845 break; 2846 } 2847 return 0; 2848 } 2849 2850 static void build_cpl_pass_accept_req(struct sk_buff *skb, int stid , u8 tos) 2851 { 2852 u32 l2info; 2853 u16 vlantag, len, hdr_len; 2854 u8 intf; 2855 struct cpl_rx_pkt *cpl = cplhdr(skb); 2856 struct cpl_pass_accept_req *req; 2857 struct tcp_options_received tmp_opt; 2858 2859 /* Store values from cpl_rx_pkt in temporary location. */ 2860 vlantag = cpl->vlan; 2861 len = cpl->len; 2862 l2info = cpl->l2info; 2863 hdr_len = cpl->hdr_len; 2864 intf = cpl->iff; 2865 2866 __skb_pull(skb, sizeof(*req) + sizeof(struct rss_header)); 2867 2868 /* 2869 * We need to parse the TCP options from SYN packet. 2870 * to generate cpl_pass_accept_req. 2871 */ 2872 memset(&tmp_opt, 0, sizeof(tmp_opt)); 2873 tcp_clear_options(&tmp_opt); 2874 tcp_parse_options(skb, &tmp_opt, 0, 0, NULL); 2875 2876 req = (struct cpl_pass_accept_req *)__skb_push(skb, sizeof(*req)); 2877 memset(req, 0, sizeof(*req)); 2878 req->l2info = cpu_to_be16(V_SYN_INTF(intf) | 2879 V_SYN_MAC_IDX(G_RX_MACIDX(htonl(l2info))) | 2880 F_SYN_XACT_MATCH); 2881 req->hdr_len = cpu_to_be32(V_SYN_RX_CHAN(G_RX_CHAN(htonl(l2info))) | 2882 V_TCP_HDR_LEN(G_RX_TCPHDR_LEN(htons(hdr_len))) | 2883 V_IP_HDR_LEN(G_RX_IPHDR_LEN(htons(hdr_len))) | 2884 V_ETH_HDR_LEN(G_RX_ETHHDR_LEN(htonl(l2info)))); 2885 req->vlan = vlantag; 2886 req->len = len; 2887 req->tos_stid = cpu_to_be32(PASS_OPEN_TID(stid) | 2888 PASS_OPEN_TOS(tos)); 2889 req->tcpopt.mss = htons(tmp_opt.mss_clamp); 2890 if (tmp_opt.wscale_ok) 2891 req->tcpopt.wsf = tmp_opt.snd_wscale; 2892 req->tcpopt.tstamp = tmp_opt.saw_tstamp; 2893 if (tmp_opt.sack_ok) 2894 req->tcpopt.sack = 1; 2895 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 0)); 2896 return; 2897 } 2898 2899 static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, 2900 __be32 laddr, __be16 lport, 2901 __be32 raddr, __be16 rport, 2902 u32 rcv_isn, u32 filter, u16 window, 2903 u32 rss_qid, u8 port_id) 2904 { 2905 struct sk_buff *req_skb; 2906 struct fw_ofld_connection_wr *req; 2907 struct cpl_pass_accept_req *cpl = cplhdr(skb); 2908 2909 req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); 2910 req = (struct fw_ofld_connection_wr *)__skb_put(req_skb, sizeof(*req)); 2911 memset(req, 0, sizeof(*req)); 2912 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL(1)); 2913 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 2914 req->le.version_cpl = htonl(F_FW_OFLD_CONNECTION_WR_CPL); 2915 req->le.filter = filter; 2916 req->le.lport = lport; 2917 req->le.pport = rport; 2918 req->le.u.ipv4.lip = laddr; 2919 req->le.u.ipv4.pip = raddr; 2920 req->tcb.rcv_nxt = htonl(rcv_isn + 1); 2921 req->tcb.rcv_adv = htons(window); 2922 req->tcb.t_state_to_astid = 2923 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_RECV) | 2924 V_FW_OFLD_CONNECTION_WR_RCV_SCALE(cpl->tcpopt.wsf) | 2925 V_FW_OFLD_CONNECTION_WR_ASTID( 2926 GET_PASS_OPEN_TID(ntohl(cpl->tos_stid)))); 2927 2928 /* 2929 * We store the qid in opt2 which will be used by the firmware 2930 * to send us the wr response. 2931 */ 2932 req->tcb.opt2 = htonl(V_RSS_QUEUE(rss_qid)); 2933 2934 /* 2935 * We initialize the MSS index in TCB to 0xF. 2936 * So that when driver sends cpl_pass_accept_rpl 2937 * TCB picks up the correct value. If this was 0 2938 * TP will ignore any value > 0 for MSS index. 2939 */ 2940 req->tcb.opt0 = cpu_to_be64(V_MSS_IDX(0xF)); 2941 req->cookie = cpu_to_be64((u64)skb); 2942 2943 set_wr_txq(req_skb, CPL_PRIORITY_CONTROL, port_id); 2944 cxgb4_ofld_send(dev->rdev.lldi.ports[0], req_skb); 2945 } 2946 2947 /* 2948 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt 2949 * messages when a filter is being used instead of server to 2950 * redirect a syn packet. When packets hit filter they are redirected 2951 * to the offload queue and driver tries to establish the connection 2952 * using firmware work request. 2953 */ 2954 static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) 2955 { 2956 int stid; 2957 unsigned int filter; 2958 struct ethhdr *eh = NULL; 2959 struct vlan_ethhdr *vlan_eh = NULL; 2960 struct iphdr *iph; 2961 struct tcphdr *tcph; 2962 struct rss_header *rss = (void *)skb->data; 2963 struct cpl_rx_pkt *cpl = (void *)skb->data; 2964 struct cpl_pass_accept_req *req = (void *)(rss + 1); 2965 struct l2t_entry *e; 2966 struct dst_entry *dst; 2967 struct rtable *rt; 2968 struct c4iw_ep *lep; 2969 u16 window; 2970 struct port_info *pi; 2971 struct net_device *pdev; 2972 u16 rss_qid; 2973 int step; 2974 u32 tx_chan; 2975 struct neighbour *neigh; 2976 2977 /* Drop all non-SYN packets */ 2978 if (!(cpl->l2info & cpu_to_be32(F_RXF_SYN))) 2979 goto reject; 2980 2981 /* 2982 * Drop all packets which did not hit the filter. 2983 * Unlikely to happen. 2984 */ 2985 if (!(rss->filter_hit && rss->filter_tid)) 2986 goto reject; 2987 2988 /* 2989 * Calculate the server tid from filter hit index from cpl_rx_pkt. 2990 */ 2991 stid = cpu_to_be32(rss->hash_val) - dev->rdev.lldi.tids->sftid_base 2992 + dev->rdev.lldi.tids->nstids; 2993 2994 lep = (struct c4iw_ep *)lookup_stid(dev->rdev.lldi.tids, stid); 2995 if (!lep) { 2996 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 2997 goto reject; 2998 } 2999 3000 if (G_RX_ETHHDR_LEN(ntohl(cpl->l2info)) == ETH_HLEN) { 3001 eh = (struct ethhdr *)(req + 1); 3002 iph = (struct iphdr *)(eh + 1); 3003 } else { 3004 vlan_eh = (struct vlan_ethhdr *)(req + 1); 3005 iph = (struct iphdr *)(vlan_eh + 1); 3006 skb->vlan_tci = ntohs(cpl->vlan); 3007 } 3008 3009 if (iph->version != 0x4) 3010 goto reject; 3011 3012 tcph = (struct tcphdr *)(iph + 1); 3013 skb_set_network_header(skb, (void *)iph - (void *)rss); 3014 skb_set_transport_header(skb, (void *)tcph - (void *)rss); 3015 skb_get(skb); 3016 3017 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__, 3018 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3019 ntohs(tcph->source), iph->tos); 3020 3021 rt = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3022 iph->tos); 3023 if (!rt) { 3024 pr_err("%s - failed to find dst entry!\n", 3025 __func__); 3026 goto reject; 3027 } 3028 dst = &rt->dst; 3029 neigh = dst_neigh_lookup_skb(dst, skb); 3030 3031 if (neigh->dev->flags & IFF_LOOPBACK) { 3032 pdev = ip_dev_find(&init_net, iph->daddr); 3033 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3034 pdev, 0); 3035 pi = (struct port_info *)netdev_priv(pdev); 3036 tx_chan = cxgb4_port_chan(pdev); 3037 dev_put(pdev); 3038 } else { 3039 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3040 neigh->dev, 0); 3041 pi = (struct port_info *)netdev_priv(neigh->dev); 3042 tx_chan = cxgb4_port_chan(neigh->dev); 3043 } 3044 if (!e) { 3045 pr_err("%s - failed to allocate l2t entry!\n", 3046 __func__); 3047 goto free_dst; 3048 } 3049 3050 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; 3051 rss_qid = dev->rdev.lldi.rxq_ids[pi->port_id * step]; 3052 window = htons(tcph->window); 3053 3054 /* Calcuate filter portion for LE region. */ 3055 filter = cpu_to_be32(select_ntuple(dev, dst, e)); 3056 3057 /* 3058 * Synthesize the cpl_pass_accept_req. We have everything except the 3059 * TID. Once firmware sends a reply with TID we update the TID field 3060 * in cpl and pass it through the regular cpl_pass_accept_req path. 3061 */ 3062 build_cpl_pass_accept_req(skb, stid, iph->tos); 3063 send_fw_pass_open_req(dev, skb, iph->daddr, tcph->dest, iph->saddr, 3064 tcph->source, ntohl(tcph->seq), filter, window, 3065 rss_qid, pi->port_id); 3066 cxgb4_l2t_release(e); 3067 free_dst: 3068 dst_release(dst); 3069 reject: 3070 return 0; 3071 } 3072 3073 /* 3074 * These are the real handlers that are called from a 3075 * work queue. 3076 */ 3077 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 3078 [CPL_ACT_ESTABLISH] = act_establish, 3079 [CPL_ACT_OPEN_RPL] = act_open_rpl, 3080 [CPL_RX_DATA] = rx_data, 3081 [CPL_ABORT_RPL_RSS] = abort_rpl, 3082 [CPL_ABORT_RPL] = abort_rpl, 3083 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 3084 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 3085 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 3086 [CPL_PASS_ESTABLISH] = pass_establish, 3087 [CPL_PEER_CLOSE] = peer_close, 3088 [CPL_ABORT_REQ_RSS] = peer_abort, 3089 [CPL_CLOSE_CON_RPL] = close_con_rpl, 3090 [CPL_RDMA_TERMINATE] = terminate, 3091 [CPL_FW4_ACK] = fw4_ack, 3092 [CPL_FW6_MSG] = deferred_fw6_msg, 3093 [CPL_RX_PKT] = rx_pkt 3094 }; 3095 3096 static void process_timeout(struct c4iw_ep *ep) 3097 { 3098 struct c4iw_qp_attributes attrs; 3099 int abort = 1; 3100 3101 mutex_lock(&ep->com.mutex); 3102 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 3103 ep->com.state); 3104 set_bit(TIMEDOUT, &ep->com.history); 3105 switch (ep->com.state) { 3106 case MPA_REQ_SENT: 3107 __state_set(&ep->com, ABORTING); 3108 connect_reply_upcall(ep, -ETIMEDOUT); 3109 break; 3110 case MPA_REQ_WAIT: 3111 __state_set(&ep->com, ABORTING); 3112 break; 3113 case CLOSING: 3114 case MORIBUND: 3115 if (ep->com.cm_id && ep->com.qp) { 3116 attrs.next_state = C4IW_QP_STATE_ERROR; 3117 c4iw_modify_qp(ep->com.qp->rhp, 3118 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 3119 &attrs, 1); 3120 } 3121 __state_set(&ep->com, ABORTING); 3122 break; 3123 default: 3124 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3125 __func__, ep, ep->hwtid, ep->com.state); 3126 abort = 0; 3127 } 3128 mutex_unlock(&ep->com.mutex); 3129 if (abort) 3130 abort_connection(ep, NULL, GFP_KERNEL); 3131 c4iw_put_ep(&ep->com); 3132 } 3133 3134 static void process_timedout_eps(void) 3135 { 3136 struct c4iw_ep *ep; 3137 3138 spin_lock_irq(&timeout_lock); 3139 while (!list_empty(&timeout_list)) { 3140 struct list_head *tmp; 3141 3142 tmp = timeout_list.next; 3143 list_del(tmp); 3144 spin_unlock_irq(&timeout_lock); 3145 ep = list_entry(tmp, struct c4iw_ep, entry); 3146 process_timeout(ep); 3147 spin_lock_irq(&timeout_lock); 3148 } 3149 spin_unlock_irq(&timeout_lock); 3150 } 3151 3152 static void process_work(struct work_struct *work) 3153 { 3154 struct sk_buff *skb = NULL; 3155 struct c4iw_dev *dev; 3156 struct cpl_act_establish *rpl; 3157 unsigned int opcode; 3158 int ret; 3159 3160 while ((skb = skb_dequeue(&rxq))) { 3161 rpl = cplhdr(skb); 3162 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3163 opcode = rpl->ot.opcode; 3164 3165 BUG_ON(!work_handlers[opcode]); 3166 ret = work_handlers[opcode](dev, skb); 3167 if (!ret) 3168 kfree_skb(skb); 3169 } 3170 process_timedout_eps(); 3171 } 3172 3173 static DECLARE_WORK(skb_work, process_work); 3174 3175 static void ep_timeout(unsigned long arg) 3176 { 3177 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 3178 3179 spin_lock(&timeout_lock); 3180 list_add_tail(&ep->entry, &timeout_list); 3181 spin_unlock(&timeout_lock); 3182 queue_work(workq, &skb_work); 3183 } 3184 3185 /* 3186 * All the CM events are handled on a work queue to have a safe context. 3187 */ 3188 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 3189 { 3190 3191 /* 3192 * Save dev in the skb->cb area. 3193 */ 3194 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 3195 3196 /* 3197 * Queue the skb and schedule the worker thread. 3198 */ 3199 skb_queue_tail(&rxq, skb); 3200 queue_work(workq, &skb_work); 3201 return 0; 3202 } 3203 3204 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 3205 { 3206 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 3207 3208 if (rpl->status != CPL_ERR_NONE) { 3209 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 3210 "for tid %u\n", rpl->status, GET_TID(rpl)); 3211 } 3212 kfree_skb(skb); 3213 return 0; 3214 } 3215 3216 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3217 { 3218 struct cpl_fw6_msg *rpl = cplhdr(skb); 3219 struct c4iw_wr_wait *wr_waitp; 3220 int ret; 3221 3222 PDBG("%s type %u\n", __func__, rpl->type); 3223 3224 switch (rpl->type) { 3225 case FW6_TYPE_WR_RPL: 3226 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 3227 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 3228 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 3229 if (wr_waitp) 3230 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 3231 kfree_skb(skb); 3232 break; 3233 case FW6_TYPE_CQE: 3234 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3235 sched(dev, skb); 3236 break; 3237 default: 3238 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 3239 rpl->type); 3240 kfree_skb(skb); 3241 break; 3242 } 3243 return 0; 3244 } 3245 3246 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 3247 { 3248 struct cpl_abort_req_rss *req = cplhdr(skb); 3249 struct c4iw_ep *ep; 3250 struct tid_info *t = dev->rdev.lldi.tids; 3251 unsigned int tid = GET_TID(req); 3252 3253 ep = lookup_tid(t, tid); 3254 if (!ep) { 3255 printk(KERN_WARNING MOD 3256 "Abort on non-existent endpoint, tid %d\n", tid); 3257 kfree_skb(skb); 3258 return 0; 3259 } 3260 if (is_neg_adv_abort(req->status)) { 3261 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 3262 ep->hwtid); 3263 kfree_skb(skb); 3264 return 0; 3265 } 3266 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 3267 ep->com.state); 3268 3269 /* 3270 * Wake up any threads in rdma_init() or rdma_fini(). 3271 */ 3272 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3273 sched(dev, skb); 3274 return 0; 3275 } 3276 3277 /* 3278 * Most upcalls from the T4 Core go to sched() to 3279 * schedule the processing on a work queue. 3280 */ 3281 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 3282 [CPL_ACT_ESTABLISH] = sched, 3283 [CPL_ACT_OPEN_RPL] = sched, 3284 [CPL_RX_DATA] = sched, 3285 [CPL_ABORT_RPL_RSS] = sched, 3286 [CPL_ABORT_RPL] = sched, 3287 [CPL_PASS_OPEN_RPL] = sched, 3288 [CPL_CLOSE_LISTSRV_RPL] = sched, 3289 [CPL_PASS_ACCEPT_REQ] = sched, 3290 [CPL_PASS_ESTABLISH] = sched, 3291 [CPL_PEER_CLOSE] = sched, 3292 [CPL_CLOSE_CON_RPL] = sched, 3293 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 3294 [CPL_RDMA_TERMINATE] = sched, 3295 [CPL_FW4_ACK] = sched, 3296 [CPL_SET_TCB_RPL] = set_tcb_rpl, 3297 [CPL_FW6_MSG] = fw6_msg, 3298 [CPL_RX_PKT] = sched 3299 }; 3300 3301 int __init c4iw_cm_init(void) 3302 { 3303 spin_lock_init(&timeout_lock); 3304 skb_queue_head_init(&rxq); 3305 3306 workq = create_singlethread_workqueue("iw_cxgb4"); 3307 if (!workq) 3308 return -ENOMEM; 3309 3310 return 0; 3311 } 3312 3313 void __exit c4iw_cm_term(void) 3314 { 3315 WARN_ON(!list_empty(&timeout_list)); 3316 flush_workqueue(workq); 3317 destroy_workqueue(workq); 3318 } 3319