1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 #include <linux/if_vlan.h> 42 43 #include <net/neighbour.h> 44 #include <net/netevent.h> 45 #include <net/route.h> 46 #include <net/tcp.h> 47 #include <net/ip6_route.h> 48 #include <net/addrconf.h> 49 50 #include "iw_cxgb4.h" 51 52 static char *states[] = { 53 "idle", 54 "listen", 55 "connecting", 56 "mpa_wait_req", 57 "mpa_req_sent", 58 "mpa_req_rcvd", 59 "mpa_rep_sent", 60 "fpdu_mode", 61 "aborting", 62 "closing", 63 "moribund", 64 "dead", 65 NULL, 66 }; 67 68 static int nocong; 69 module_param(nocong, int, 0644); 70 MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)"); 71 72 static int enable_ecn; 73 module_param(enable_ecn, int, 0644); 74 MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); 75 76 static int dack_mode = 1; 77 module_param(dack_mode, int, 0644); 78 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 79 80 int c4iw_max_read_depth = 8; 81 module_param(c4iw_max_read_depth, int, 0644); 82 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 83 84 static int enable_tcp_timestamps; 85 module_param(enable_tcp_timestamps, int, 0644); 86 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 87 88 static int enable_tcp_sack; 89 module_param(enable_tcp_sack, int, 0644); 90 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 91 92 static int enable_tcp_window_scaling = 1; 93 module_param(enable_tcp_window_scaling, int, 0644); 94 MODULE_PARM_DESC(enable_tcp_window_scaling, 95 "Enable tcp window scaling (default=1)"); 96 97 int c4iw_debug; 98 module_param(c4iw_debug, int, 0644); 99 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 100 101 static int peer2peer; 102 module_param(peer2peer, int, 0644); 103 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 104 105 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 106 module_param(p2p_type, int, 0644); 107 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 108 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 109 110 static int ep_timeout_secs = 60; 111 module_param(ep_timeout_secs, int, 0644); 112 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 113 "in seconds (default=60)"); 114 115 static int mpa_rev = 1; 116 module_param(mpa_rev, int, 0644); 117 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 118 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 119 " compliant (default=1)"); 120 121 static int markers_enabled; 122 module_param(markers_enabled, int, 0644); 123 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 124 125 static int crc_enabled = 1; 126 module_param(crc_enabled, int, 0644); 127 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 128 129 static int rcv_win = 256 * 1024; 130 module_param(rcv_win, int, 0644); 131 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 132 133 static int snd_win = 128 * 1024; 134 module_param(snd_win, int, 0644); 135 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 136 137 static struct workqueue_struct *workq; 138 139 static struct sk_buff_head rxq; 140 141 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 142 static void ep_timeout(unsigned long arg); 143 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 144 145 static LIST_HEAD(timeout_list); 146 static spinlock_t timeout_lock; 147 148 static void deref_qp(struct c4iw_ep *ep) 149 { 150 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 151 clear_bit(QP_REFERENCED, &ep->com.flags); 152 } 153 154 static void ref_qp(struct c4iw_ep *ep) 155 { 156 set_bit(QP_REFERENCED, &ep->com.flags); 157 c4iw_qp_add_ref(&ep->com.qp->ibqp); 158 } 159 160 static void start_ep_timer(struct c4iw_ep *ep) 161 { 162 PDBG("%s ep %p\n", __func__, ep); 163 if (timer_pending(&ep->timer)) { 164 pr_err("%s timer already started! ep %p\n", 165 __func__, ep); 166 return; 167 } 168 clear_bit(TIMEOUT, &ep->com.flags); 169 c4iw_get_ep(&ep->com); 170 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 171 ep->timer.data = (unsigned long)ep; 172 ep->timer.function = ep_timeout; 173 add_timer(&ep->timer); 174 } 175 176 static void stop_ep_timer(struct c4iw_ep *ep) 177 { 178 PDBG("%s ep %p stopping\n", __func__, ep); 179 del_timer_sync(&ep->timer); 180 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) 181 c4iw_put_ep(&ep->com); 182 } 183 184 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 185 struct l2t_entry *l2e) 186 { 187 int error = 0; 188 189 if (c4iw_fatal_error(rdev)) { 190 kfree_skb(skb); 191 PDBG("%s - device in error state - dropping\n", __func__); 192 return -EIO; 193 } 194 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 195 if (error < 0) 196 kfree_skb(skb); 197 return error < 0 ? error : 0; 198 } 199 200 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 201 { 202 int error = 0; 203 204 if (c4iw_fatal_error(rdev)) { 205 kfree_skb(skb); 206 PDBG("%s - device in error state - dropping\n", __func__); 207 return -EIO; 208 } 209 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 210 if (error < 0) 211 kfree_skb(skb); 212 return error < 0 ? error : 0; 213 } 214 215 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 216 { 217 struct cpl_tid_release *req; 218 219 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 220 if (!skb) 221 return; 222 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 223 INIT_TP_WR(req, hwtid); 224 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 225 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 226 c4iw_ofld_send(rdev, skb); 227 return; 228 } 229 230 static void set_emss(struct c4iw_ep *ep, u16 opt) 231 { 232 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 233 ep->mss = ep->emss; 234 if (GET_TCPOPT_TSTAMP(opt)) 235 ep->emss -= 12; 236 if (ep->emss < 128) 237 ep->emss = 128; 238 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 239 ep->mss, ep->emss); 240 } 241 242 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 243 { 244 enum c4iw_ep_state state; 245 246 mutex_lock(&epc->mutex); 247 state = epc->state; 248 mutex_unlock(&epc->mutex); 249 return state; 250 } 251 252 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 253 { 254 epc->state = new; 255 } 256 257 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 258 { 259 mutex_lock(&epc->mutex); 260 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 261 __state_set(epc, new); 262 mutex_unlock(&epc->mutex); 263 return; 264 } 265 266 static void *alloc_ep(int size, gfp_t gfp) 267 { 268 struct c4iw_ep_common *epc; 269 270 epc = kzalloc(size, gfp); 271 if (epc) { 272 kref_init(&epc->kref); 273 mutex_init(&epc->mutex); 274 c4iw_init_wr_wait(&epc->wr_wait); 275 } 276 PDBG("%s alloc ep %p\n", __func__, epc); 277 return epc; 278 } 279 280 void _c4iw_free_ep(struct kref *kref) 281 { 282 struct c4iw_ep *ep; 283 284 ep = container_of(kref, struct c4iw_ep, com.kref); 285 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 286 if (test_bit(QP_REFERENCED, &ep->com.flags)) 287 deref_qp(ep); 288 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 289 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 290 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 291 dst_release(ep->dst); 292 cxgb4_l2t_release(ep->l2t); 293 } 294 kfree(ep); 295 } 296 297 static void release_ep_resources(struct c4iw_ep *ep) 298 { 299 set_bit(RELEASE_RESOURCES, &ep->com.flags); 300 c4iw_put_ep(&ep->com); 301 } 302 303 static int status2errno(int status) 304 { 305 switch (status) { 306 case CPL_ERR_NONE: 307 return 0; 308 case CPL_ERR_CONN_RESET: 309 return -ECONNRESET; 310 case CPL_ERR_ARP_MISS: 311 return -EHOSTUNREACH; 312 case CPL_ERR_CONN_TIMEDOUT: 313 return -ETIMEDOUT; 314 case CPL_ERR_TCAM_FULL: 315 return -ENOMEM; 316 case CPL_ERR_CONN_EXIST: 317 return -EADDRINUSE; 318 default: 319 return -EIO; 320 } 321 } 322 323 /* 324 * Try and reuse skbs already allocated... 325 */ 326 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 327 { 328 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 329 skb_trim(skb, 0); 330 skb_get(skb); 331 skb_reset_transport_header(skb); 332 } else { 333 skb = alloc_skb(len, gfp); 334 } 335 t4_set_arp_err_handler(skb, NULL, NULL); 336 return skb; 337 } 338 339 static struct net_device *get_real_dev(struct net_device *egress_dev) 340 { 341 struct net_device *phys_dev = egress_dev; 342 if (egress_dev->priv_flags & IFF_802_1Q_VLAN) 343 phys_dev = vlan_dev_real_dev(egress_dev); 344 return phys_dev; 345 } 346 347 static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev) 348 { 349 int i; 350 351 egress_dev = get_real_dev(egress_dev); 352 for (i = 0; i < dev->rdev.lldi.nports; i++) 353 if (dev->rdev.lldi.ports[i] == egress_dev) 354 return 1; 355 return 0; 356 } 357 358 static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip, 359 __u8 *peer_ip, __be16 local_port, 360 __be16 peer_port, u8 tos, 361 __u32 sin6_scope_id) 362 { 363 struct dst_entry *dst = NULL; 364 365 if (IS_ENABLED(CONFIG_IPV6)) { 366 struct flowi6 fl6; 367 368 memset(&fl6, 0, sizeof(fl6)); 369 memcpy(&fl6.daddr, peer_ip, 16); 370 memcpy(&fl6.saddr, local_ip, 16); 371 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 372 fl6.flowi6_oif = sin6_scope_id; 373 dst = ip6_route_output(&init_net, NULL, &fl6); 374 if (!dst) 375 goto out; 376 if (!our_interface(dev, ip6_dst_idev(dst)->dev) && 377 !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) { 378 dst_release(dst); 379 dst = NULL; 380 } 381 } 382 383 out: 384 return dst; 385 } 386 387 static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip, 388 __be32 peer_ip, __be16 local_port, 389 __be16 peer_port, u8 tos) 390 { 391 struct rtable *rt; 392 struct flowi4 fl4; 393 struct neighbour *n; 394 395 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 396 peer_port, local_port, IPPROTO_TCP, 397 tos, 0); 398 if (IS_ERR(rt)) 399 return NULL; 400 n = dst_neigh_lookup(&rt->dst, &peer_ip); 401 if (!n) 402 return NULL; 403 if (!our_interface(dev, n->dev)) { 404 dst_release(&rt->dst); 405 return NULL; 406 } 407 neigh_release(n); 408 return &rt->dst; 409 } 410 411 static void arp_failure_discard(void *handle, struct sk_buff *skb) 412 { 413 PDBG("%s c4iw_dev %p\n", __func__, handle); 414 kfree_skb(skb); 415 } 416 417 /* 418 * Handle an ARP failure for an active open. 419 */ 420 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 421 { 422 printk(KERN_ERR MOD "ARP failure duing connect\n"); 423 kfree_skb(skb); 424 } 425 426 /* 427 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 428 * and send it along. 429 */ 430 static void abort_arp_failure(void *handle, struct sk_buff *skb) 431 { 432 struct c4iw_rdev *rdev = handle; 433 struct cpl_abort_req *req = cplhdr(skb); 434 435 PDBG("%s rdev %p\n", __func__, rdev); 436 req->cmd = CPL_ABORT_NO_RST; 437 c4iw_ofld_send(rdev, skb); 438 } 439 440 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 441 { 442 unsigned int flowclen = 80; 443 struct fw_flowc_wr *flowc; 444 int i; 445 446 skb = get_skb(skb, flowclen, GFP_KERNEL); 447 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 448 449 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 450 FW_FLOWC_WR_NPARAMS(8)); 451 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 452 16)) | FW_WR_FLOWID(ep->hwtid)); 453 454 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 455 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 456 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 457 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 458 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 459 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 460 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 461 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 462 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 463 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 464 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 465 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 466 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 467 flowc->mnemval[6].val = cpu_to_be32(snd_win); 468 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 469 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 470 /* Pad WR to 16 byte boundary */ 471 flowc->mnemval[8].mnemonic = 0; 472 flowc->mnemval[8].val = 0; 473 for (i = 0; i < 9; i++) { 474 flowc->mnemval[i].r4[0] = 0; 475 flowc->mnemval[i].r4[1] = 0; 476 flowc->mnemval[i].r4[2] = 0; 477 } 478 479 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 480 c4iw_ofld_send(&ep->com.dev->rdev, skb); 481 } 482 483 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 484 { 485 struct cpl_close_con_req *req; 486 struct sk_buff *skb; 487 int wrlen = roundup(sizeof *req, 16); 488 489 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 490 skb = get_skb(NULL, wrlen, gfp); 491 if (!skb) { 492 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 493 return -ENOMEM; 494 } 495 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 496 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 497 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 498 memset(req, 0, wrlen); 499 INIT_TP_WR(req, ep->hwtid); 500 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 501 ep->hwtid)); 502 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 503 } 504 505 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 506 { 507 struct cpl_abort_req *req; 508 int wrlen = roundup(sizeof *req, 16); 509 510 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 511 skb = get_skb(skb, wrlen, gfp); 512 if (!skb) { 513 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 514 __func__); 515 return -ENOMEM; 516 } 517 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 518 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 519 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 520 memset(req, 0, wrlen); 521 INIT_TP_WR(req, ep->hwtid); 522 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 523 req->cmd = CPL_ABORT_SEND_RST; 524 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 525 } 526 527 #define VLAN_NONE 0xfff 528 #define FILTER_SEL_VLAN_NONE 0xffff 529 #define FILTER_SEL_WIDTH_P_FC (3+1) /* port uses 3 bits, FCoE one bit */ 530 #define FILTER_SEL_WIDTH_VIN_P_FC \ 531 (6 + 7 + FILTER_SEL_WIDTH_P_FC) /* 6 bits are unused, VF uses 7 bits*/ 532 #define FILTER_SEL_WIDTH_TAG_P_FC \ 533 (3 + FILTER_SEL_WIDTH_VIN_P_FC) /* PF uses 3 bits */ 534 #define FILTER_SEL_WIDTH_VLD_TAG_P_FC (1 + FILTER_SEL_WIDTH_TAG_P_FC) 535 536 static unsigned int select_ntuple(struct c4iw_dev *dev, struct dst_entry *dst, 537 struct l2t_entry *l2t) 538 { 539 unsigned int ntuple = 0; 540 u32 viid; 541 542 switch (dev->rdev.lldi.filt_mode) { 543 544 /* default filter mode */ 545 case HW_TPL_FR_MT_PR_IV_P_FC: 546 if (l2t->vlan == VLAN_NONE) 547 ntuple |= FILTER_SEL_VLAN_NONE << FILTER_SEL_WIDTH_P_FC; 548 else { 549 ntuple |= l2t->vlan << FILTER_SEL_WIDTH_P_FC; 550 ntuple |= 1 << FILTER_SEL_WIDTH_TAG_P_FC; 551 } 552 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 553 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 554 break; 555 case HW_TPL_FR_MT_PR_OV_P_FC: { 556 viid = cxgb4_port_viid(l2t->neigh->dev); 557 558 ntuple |= FW_VIID_VIN_GET(viid) << FILTER_SEL_WIDTH_P_FC; 559 ntuple |= FW_VIID_PFN_GET(viid) << FILTER_SEL_WIDTH_VIN_P_FC; 560 ntuple |= FW_VIID_VIVLD_GET(viid) << FILTER_SEL_WIDTH_TAG_P_FC; 561 ntuple |= l2t->lport << S_PORT | IPPROTO_TCP << 562 FILTER_SEL_WIDTH_VLD_TAG_P_FC; 563 break; 564 } 565 default: 566 break; 567 } 568 return ntuple; 569 } 570 571 static int send_connect(struct c4iw_ep *ep) 572 { 573 struct cpl_act_open_req *req; 574 struct cpl_t5_act_open_req *t5_req; 575 struct cpl_act_open_req6 *req6; 576 struct cpl_t5_act_open_req6 *t5_req6; 577 struct sk_buff *skb; 578 u64 opt0; 579 u32 opt2; 580 unsigned int mtu_idx; 581 int wscale; 582 int wrlen; 583 int sizev4 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 584 sizeof(struct cpl_act_open_req) : 585 sizeof(struct cpl_t5_act_open_req); 586 int sizev6 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 587 sizeof(struct cpl_act_open_req6) : 588 sizeof(struct cpl_t5_act_open_req6); 589 struct sockaddr_in *la = (struct sockaddr_in *)&ep->com.local_addr; 590 struct sockaddr_in *ra = (struct sockaddr_in *)&ep->com.remote_addr; 591 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 592 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 593 594 wrlen = (ep->com.remote_addr.ss_family == AF_INET) ? 595 roundup(sizev4, 16) : 596 roundup(sizev6, 16); 597 598 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 599 600 skb = get_skb(NULL, wrlen, GFP_KERNEL); 601 if (!skb) { 602 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 603 __func__); 604 return -ENOMEM; 605 } 606 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 607 608 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 609 wscale = compute_wscale(rcv_win); 610 opt0 = (nocong ? NO_CONG(1) : 0) | 611 KEEP_ALIVE(1) | 612 DELACK(1) | 613 WND_SCALE(wscale) | 614 MSS_IDX(mtu_idx) | 615 L2T_IDX(ep->l2t->idx) | 616 TX_CHAN(ep->tx_chan) | 617 SMAC_SEL(ep->smac_idx) | 618 DSCP(ep->tos) | 619 ULP_MODE(ULP_MODE_TCPDDP) | 620 RCV_BUFSIZ(rcv_win>>10); 621 opt2 = RX_CHANNEL(0) | 622 CCTRL_ECN(enable_ecn) | 623 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 624 if (enable_tcp_timestamps) 625 opt2 |= TSTAMPS_EN(1); 626 if (enable_tcp_sack) 627 opt2 |= SACK_EN(1); 628 if (wscale && enable_tcp_window_scaling) 629 opt2 |= WND_SCALE_EN(1); 630 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 631 632 if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { 633 if (ep->com.remote_addr.ss_family == AF_INET) { 634 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 635 INIT_TP_WR(req, 0); 636 OPCODE_TID(req) = cpu_to_be32( 637 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 638 ((ep->rss_qid << 14) | ep->atid))); 639 req->local_port = la->sin_port; 640 req->peer_port = ra->sin_port; 641 req->local_ip = la->sin_addr.s_addr; 642 req->peer_ip = ra->sin_addr.s_addr; 643 req->opt0 = cpu_to_be64(opt0); 644 req->params = cpu_to_be32(select_ntuple(ep->com.dev, 645 ep->dst, ep->l2t)); 646 req->opt2 = cpu_to_be32(opt2); 647 } else { 648 req6 = (struct cpl_act_open_req6 *)skb_put(skb, wrlen); 649 650 INIT_TP_WR(req6, 0); 651 OPCODE_TID(req6) = cpu_to_be32( 652 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 653 ((ep->rss_qid<<14)|ep->atid))); 654 req6->local_port = la6->sin6_port; 655 req6->peer_port = ra6->sin6_port; 656 req6->local_ip_hi = *((__be64 *) 657 (la6->sin6_addr.s6_addr)); 658 req6->local_ip_lo = *((__be64 *) 659 (la6->sin6_addr.s6_addr + 8)); 660 req6->peer_ip_hi = *((__be64 *) 661 (ra6->sin6_addr.s6_addr)); 662 req6->peer_ip_lo = *((__be64 *) 663 (ra6->sin6_addr.s6_addr + 8)); 664 req6->opt0 = cpu_to_be64(opt0); 665 req6->params = cpu_to_be32( 666 select_ntuple(ep->com.dev, ep->dst, 667 ep->l2t)); 668 req6->opt2 = cpu_to_be32(opt2); 669 } 670 } else { 671 if (ep->com.remote_addr.ss_family == AF_INET) { 672 t5_req = (struct cpl_t5_act_open_req *) 673 skb_put(skb, wrlen); 674 INIT_TP_WR(t5_req, 0); 675 OPCODE_TID(t5_req) = cpu_to_be32( 676 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 677 ((ep->rss_qid << 14) | ep->atid))); 678 t5_req->local_port = la->sin_port; 679 t5_req->peer_port = ra->sin_port; 680 t5_req->local_ip = la->sin_addr.s_addr; 681 t5_req->peer_ip = ra->sin_addr.s_addr; 682 t5_req->opt0 = cpu_to_be64(opt0); 683 t5_req->params = cpu_to_be64(V_FILTER_TUPLE( 684 select_ntuple(ep->com.dev, 685 ep->dst, ep->l2t))); 686 t5_req->opt2 = cpu_to_be32(opt2); 687 } else { 688 t5_req6 = (struct cpl_t5_act_open_req6 *) 689 skb_put(skb, wrlen); 690 INIT_TP_WR(t5_req6, 0); 691 OPCODE_TID(t5_req6) = cpu_to_be32( 692 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 693 ((ep->rss_qid<<14)|ep->atid))); 694 t5_req6->local_port = la6->sin6_port; 695 t5_req6->peer_port = ra6->sin6_port; 696 t5_req6->local_ip_hi = *((__be64 *) 697 (la6->sin6_addr.s6_addr)); 698 t5_req6->local_ip_lo = *((__be64 *) 699 (la6->sin6_addr.s6_addr + 8)); 700 t5_req6->peer_ip_hi = *((__be64 *) 701 (ra6->sin6_addr.s6_addr)); 702 t5_req6->peer_ip_lo = *((__be64 *) 703 (ra6->sin6_addr.s6_addr + 8)); 704 t5_req6->opt0 = cpu_to_be64(opt0); 705 t5_req6->params = (__force __be64)cpu_to_be32( 706 select_ntuple(ep->com.dev, ep->dst, ep->l2t)); 707 t5_req6->opt2 = cpu_to_be32(opt2); 708 } 709 } 710 711 set_bit(ACT_OPEN_REQ, &ep->com.history); 712 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 713 } 714 715 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 716 u8 mpa_rev_to_use) 717 { 718 int mpalen, wrlen; 719 struct fw_ofld_tx_data_wr *req; 720 struct mpa_message *mpa; 721 struct mpa_v2_conn_params mpa_v2_params; 722 723 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 724 725 BUG_ON(skb_cloned(skb)); 726 727 mpalen = sizeof(*mpa) + ep->plen; 728 if (mpa_rev_to_use == 2) 729 mpalen += sizeof(struct mpa_v2_conn_params); 730 wrlen = roundup(mpalen + sizeof *req, 16); 731 skb = get_skb(skb, wrlen, GFP_KERNEL); 732 if (!skb) { 733 connect_reply_upcall(ep, -ENOMEM); 734 return; 735 } 736 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 737 738 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 739 memset(req, 0, wrlen); 740 req->op_to_immdlen = cpu_to_be32( 741 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 742 FW_WR_COMPL(1) | 743 FW_WR_IMMDLEN(mpalen)); 744 req->flowid_len16 = cpu_to_be32( 745 FW_WR_FLOWID(ep->hwtid) | 746 FW_WR_LEN16(wrlen >> 4)); 747 req->plen = cpu_to_be32(mpalen); 748 req->tunnel_to_proxy = cpu_to_be32( 749 FW_OFLD_TX_DATA_WR_FLUSH(1) | 750 FW_OFLD_TX_DATA_WR_SHOVE(1)); 751 752 mpa = (struct mpa_message *)(req + 1); 753 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 754 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 755 (markers_enabled ? MPA_MARKERS : 0) | 756 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 757 mpa->private_data_size = htons(ep->plen); 758 mpa->revision = mpa_rev_to_use; 759 if (mpa_rev_to_use == 1) { 760 ep->tried_with_mpa_v1 = 1; 761 ep->retry_with_mpa_v1 = 0; 762 } 763 764 if (mpa_rev_to_use == 2) { 765 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 766 sizeof (struct mpa_v2_conn_params)); 767 mpa_v2_params.ird = htons((u16)ep->ird); 768 mpa_v2_params.ord = htons((u16)ep->ord); 769 770 if (peer2peer) { 771 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 772 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 773 mpa_v2_params.ord |= 774 htons(MPA_V2_RDMA_WRITE_RTR); 775 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 776 mpa_v2_params.ord |= 777 htons(MPA_V2_RDMA_READ_RTR); 778 } 779 memcpy(mpa->private_data, &mpa_v2_params, 780 sizeof(struct mpa_v2_conn_params)); 781 782 if (ep->plen) 783 memcpy(mpa->private_data + 784 sizeof(struct mpa_v2_conn_params), 785 ep->mpa_pkt + sizeof(*mpa), ep->plen); 786 } else 787 if (ep->plen) 788 memcpy(mpa->private_data, 789 ep->mpa_pkt + sizeof(*mpa), ep->plen); 790 791 /* 792 * Reference the mpa skb. This ensures the data area 793 * will remain in memory until the hw acks the tx. 794 * Function fw4_ack() will deref it. 795 */ 796 skb_get(skb); 797 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 798 BUG_ON(ep->mpa_skb); 799 ep->mpa_skb = skb; 800 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 801 start_ep_timer(ep); 802 state_set(&ep->com, MPA_REQ_SENT); 803 ep->mpa_attr.initiator = 1; 804 return; 805 } 806 807 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 808 { 809 int mpalen, wrlen; 810 struct fw_ofld_tx_data_wr *req; 811 struct mpa_message *mpa; 812 struct sk_buff *skb; 813 struct mpa_v2_conn_params mpa_v2_params; 814 815 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 816 817 mpalen = sizeof(*mpa) + plen; 818 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 819 mpalen += sizeof(struct mpa_v2_conn_params); 820 wrlen = roundup(mpalen + sizeof *req, 16); 821 822 skb = get_skb(NULL, wrlen, GFP_KERNEL); 823 if (!skb) { 824 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 825 return -ENOMEM; 826 } 827 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 828 829 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 830 memset(req, 0, wrlen); 831 req->op_to_immdlen = cpu_to_be32( 832 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 833 FW_WR_COMPL(1) | 834 FW_WR_IMMDLEN(mpalen)); 835 req->flowid_len16 = cpu_to_be32( 836 FW_WR_FLOWID(ep->hwtid) | 837 FW_WR_LEN16(wrlen >> 4)); 838 req->plen = cpu_to_be32(mpalen); 839 req->tunnel_to_proxy = cpu_to_be32( 840 FW_OFLD_TX_DATA_WR_FLUSH(1) | 841 FW_OFLD_TX_DATA_WR_SHOVE(1)); 842 843 mpa = (struct mpa_message *)(req + 1); 844 memset(mpa, 0, sizeof(*mpa)); 845 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 846 mpa->flags = MPA_REJECT; 847 mpa->revision = ep->mpa_attr.version; 848 mpa->private_data_size = htons(plen); 849 850 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 851 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 852 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 853 sizeof (struct mpa_v2_conn_params)); 854 mpa_v2_params.ird = htons(((u16)ep->ird) | 855 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 856 0)); 857 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 858 (p2p_type == 859 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 860 MPA_V2_RDMA_WRITE_RTR : p2p_type == 861 FW_RI_INIT_P2PTYPE_READ_REQ ? 862 MPA_V2_RDMA_READ_RTR : 0) : 0)); 863 memcpy(mpa->private_data, &mpa_v2_params, 864 sizeof(struct mpa_v2_conn_params)); 865 866 if (ep->plen) 867 memcpy(mpa->private_data + 868 sizeof(struct mpa_v2_conn_params), pdata, plen); 869 } else 870 if (plen) 871 memcpy(mpa->private_data, pdata, plen); 872 873 /* 874 * Reference the mpa skb again. This ensures the data area 875 * will remain in memory until the hw acks the tx. 876 * Function fw4_ack() will deref it. 877 */ 878 skb_get(skb); 879 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 880 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 881 BUG_ON(ep->mpa_skb); 882 ep->mpa_skb = skb; 883 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 884 } 885 886 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 887 { 888 int mpalen, wrlen; 889 struct fw_ofld_tx_data_wr *req; 890 struct mpa_message *mpa; 891 struct sk_buff *skb; 892 struct mpa_v2_conn_params mpa_v2_params; 893 894 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 895 896 mpalen = sizeof(*mpa) + plen; 897 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 898 mpalen += sizeof(struct mpa_v2_conn_params); 899 wrlen = roundup(mpalen + sizeof *req, 16); 900 901 skb = get_skb(NULL, wrlen, GFP_KERNEL); 902 if (!skb) { 903 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 904 return -ENOMEM; 905 } 906 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 907 908 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 909 memset(req, 0, wrlen); 910 req->op_to_immdlen = cpu_to_be32( 911 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 912 FW_WR_COMPL(1) | 913 FW_WR_IMMDLEN(mpalen)); 914 req->flowid_len16 = cpu_to_be32( 915 FW_WR_FLOWID(ep->hwtid) | 916 FW_WR_LEN16(wrlen >> 4)); 917 req->plen = cpu_to_be32(mpalen); 918 req->tunnel_to_proxy = cpu_to_be32( 919 FW_OFLD_TX_DATA_WR_FLUSH(1) | 920 FW_OFLD_TX_DATA_WR_SHOVE(1)); 921 922 mpa = (struct mpa_message *)(req + 1); 923 memset(mpa, 0, sizeof(*mpa)); 924 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 925 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 926 (markers_enabled ? MPA_MARKERS : 0); 927 mpa->revision = ep->mpa_attr.version; 928 mpa->private_data_size = htons(plen); 929 930 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 931 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 932 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 933 sizeof (struct mpa_v2_conn_params)); 934 mpa_v2_params.ird = htons((u16)ep->ird); 935 mpa_v2_params.ord = htons((u16)ep->ord); 936 if (peer2peer && (ep->mpa_attr.p2p_type != 937 FW_RI_INIT_P2PTYPE_DISABLED)) { 938 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 939 940 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 941 mpa_v2_params.ord |= 942 htons(MPA_V2_RDMA_WRITE_RTR); 943 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 944 mpa_v2_params.ord |= 945 htons(MPA_V2_RDMA_READ_RTR); 946 } 947 948 memcpy(mpa->private_data, &mpa_v2_params, 949 sizeof(struct mpa_v2_conn_params)); 950 951 if (ep->plen) 952 memcpy(mpa->private_data + 953 sizeof(struct mpa_v2_conn_params), pdata, plen); 954 } else 955 if (plen) 956 memcpy(mpa->private_data, pdata, plen); 957 958 /* 959 * Reference the mpa skb. This ensures the data area 960 * will remain in memory until the hw acks the tx. 961 * Function fw4_ack() will deref it. 962 */ 963 skb_get(skb); 964 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 965 ep->mpa_skb = skb; 966 state_set(&ep->com, MPA_REP_SENT); 967 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 968 } 969 970 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 971 { 972 struct c4iw_ep *ep; 973 struct cpl_act_establish *req = cplhdr(skb); 974 unsigned int tid = GET_TID(req); 975 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 976 struct tid_info *t = dev->rdev.lldi.tids; 977 978 ep = lookup_atid(t, atid); 979 980 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 981 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 982 983 dst_confirm(ep->dst); 984 985 /* setup the hwtid for this connection */ 986 ep->hwtid = tid; 987 cxgb4_insert_tid(t, ep, tid); 988 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 989 990 ep->snd_seq = be32_to_cpu(req->snd_isn); 991 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 992 993 set_emss(ep, ntohs(req->tcp_opt)); 994 995 /* dealloc the atid */ 996 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 997 cxgb4_free_atid(t, atid); 998 set_bit(ACT_ESTAB, &ep->com.history); 999 1000 /* start MPA negotiation */ 1001 send_flowc(ep, NULL); 1002 if (ep->retry_with_mpa_v1) 1003 send_mpa_req(ep, skb, 1); 1004 else 1005 send_mpa_req(ep, skb, mpa_rev); 1006 1007 return 0; 1008 } 1009 1010 static void close_complete_upcall(struct c4iw_ep *ep) 1011 { 1012 struct iw_cm_event event; 1013 1014 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1015 memset(&event, 0, sizeof(event)); 1016 event.event = IW_CM_EVENT_CLOSE; 1017 if (ep->com.cm_id) { 1018 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 1019 ep, ep->com.cm_id, ep->hwtid); 1020 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1021 ep->com.cm_id->rem_ref(ep->com.cm_id); 1022 ep->com.cm_id = NULL; 1023 set_bit(CLOSE_UPCALL, &ep->com.history); 1024 } 1025 } 1026 1027 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 1028 { 1029 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1030 close_complete_upcall(ep); 1031 state_set(&ep->com, ABORTING); 1032 set_bit(ABORT_CONN, &ep->com.history); 1033 return send_abort(ep, skb, gfp); 1034 } 1035 1036 static void peer_close_upcall(struct c4iw_ep *ep) 1037 { 1038 struct iw_cm_event event; 1039 1040 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1041 memset(&event, 0, sizeof(event)); 1042 event.event = IW_CM_EVENT_DISCONNECT; 1043 if (ep->com.cm_id) { 1044 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 1045 ep, ep->com.cm_id, ep->hwtid); 1046 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1047 set_bit(DISCONN_UPCALL, &ep->com.history); 1048 } 1049 } 1050 1051 static void peer_abort_upcall(struct c4iw_ep *ep) 1052 { 1053 struct iw_cm_event event; 1054 1055 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1056 memset(&event, 0, sizeof(event)); 1057 event.event = IW_CM_EVENT_CLOSE; 1058 event.status = -ECONNRESET; 1059 if (ep->com.cm_id) { 1060 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 1061 ep->com.cm_id, ep->hwtid); 1062 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1063 ep->com.cm_id->rem_ref(ep->com.cm_id); 1064 ep->com.cm_id = NULL; 1065 set_bit(ABORT_UPCALL, &ep->com.history); 1066 } 1067 } 1068 1069 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1070 { 1071 struct iw_cm_event event; 1072 1073 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 1074 memset(&event, 0, sizeof(event)); 1075 event.event = IW_CM_EVENT_CONNECT_REPLY; 1076 event.status = status; 1077 memcpy(&event.local_addr, &ep->com.local_addr, 1078 sizeof(ep->com.local_addr)); 1079 memcpy(&event.remote_addr, &ep->com.remote_addr, 1080 sizeof(ep->com.remote_addr)); 1081 1082 if ((status == 0) || (status == -ECONNREFUSED)) { 1083 if (!ep->tried_with_mpa_v1) { 1084 /* this means MPA_v2 is used */ 1085 event.private_data_len = ep->plen - 1086 sizeof(struct mpa_v2_conn_params); 1087 event.private_data = ep->mpa_pkt + 1088 sizeof(struct mpa_message) + 1089 sizeof(struct mpa_v2_conn_params); 1090 } else { 1091 /* this means MPA_v1 is used */ 1092 event.private_data_len = ep->plen; 1093 event.private_data = ep->mpa_pkt + 1094 sizeof(struct mpa_message); 1095 } 1096 } 1097 1098 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 1099 ep->hwtid, status); 1100 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1101 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1102 1103 if (status < 0) { 1104 ep->com.cm_id->rem_ref(ep->com.cm_id); 1105 ep->com.cm_id = NULL; 1106 } 1107 } 1108 1109 static void connect_request_upcall(struct c4iw_ep *ep) 1110 { 1111 struct iw_cm_event event; 1112 1113 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1114 memset(&event, 0, sizeof(event)); 1115 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1116 memcpy(&event.local_addr, &ep->com.local_addr, 1117 sizeof(ep->com.local_addr)); 1118 memcpy(&event.remote_addr, &ep->com.remote_addr, 1119 sizeof(ep->com.remote_addr)); 1120 event.provider_data = ep; 1121 if (!ep->tried_with_mpa_v1) { 1122 /* this means MPA_v2 is used */ 1123 event.ord = ep->ord; 1124 event.ird = ep->ird; 1125 event.private_data_len = ep->plen - 1126 sizeof(struct mpa_v2_conn_params); 1127 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1128 sizeof(struct mpa_v2_conn_params); 1129 } else { 1130 /* this means MPA_v1 is used. Send max supported */ 1131 event.ord = c4iw_max_read_depth; 1132 event.ird = c4iw_max_read_depth; 1133 event.private_data_len = ep->plen; 1134 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1135 } 1136 if (state_read(&ep->parent_ep->com) != DEAD) { 1137 c4iw_get_ep(&ep->com); 1138 ep->parent_ep->com.cm_id->event_handler( 1139 ep->parent_ep->com.cm_id, 1140 &event); 1141 } 1142 set_bit(CONNREQ_UPCALL, &ep->com.history); 1143 c4iw_put_ep(&ep->parent_ep->com); 1144 ep->parent_ep = NULL; 1145 } 1146 1147 static void established_upcall(struct c4iw_ep *ep) 1148 { 1149 struct iw_cm_event event; 1150 1151 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1152 memset(&event, 0, sizeof(event)); 1153 event.event = IW_CM_EVENT_ESTABLISHED; 1154 event.ird = ep->ird; 1155 event.ord = ep->ord; 1156 if (ep->com.cm_id) { 1157 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1158 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1159 set_bit(ESTAB_UPCALL, &ep->com.history); 1160 } 1161 } 1162 1163 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 1164 { 1165 struct cpl_rx_data_ack *req; 1166 struct sk_buff *skb; 1167 int wrlen = roundup(sizeof *req, 16); 1168 1169 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 1170 skb = get_skb(NULL, wrlen, GFP_KERNEL); 1171 if (!skb) { 1172 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 1173 return 0; 1174 } 1175 1176 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 1177 memset(req, 0, wrlen); 1178 INIT_TP_WR(req, ep->hwtid); 1179 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1180 ep->hwtid)); 1181 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 1182 F_RX_DACK_CHANGE | 1183 V_RX_DACK_MODE(dack_mode)); 1184 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 1185 c4iw_ofld_send(&ep->com.dev->rdev, skb); 1186 return credits; 1187 } 1188 1189 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 1190 { 1191 struct mpa_message *mpa; 1192 struct mpa_v2_conn_params *mpa_v2_params; 1193 u16 plen; 1194 u16 resp_ird, resp_ord; 1195 u8 rtr_mismatch = 0, insuff_ird = 0; 1196 struct c4iw_qp_attributes attrs; 1197 enum c4iw_qp_attr_mask mask; 1198 int err; 1199 1200 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1201 1202 /* 1203 * Stop mpa timer. If it expired, then the state has 1204 * changed and we bail since ep_timeout already aborted 1205 * the connection. 1206 */ 1207 stop_ep_timer(ep); 1208 if (state_read(&ep->com) != MPA_REQ_SENT) 1209 return; 1210 1211 /* 1212 * If we get more than the supported amount of private data 1213 * then we must fail this connection. 1214 */ 1215 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1216 err = -EINVAL; 1217 goto err; 1218 } 1219 1220 /* 1221 * copy the new data into our accumulation buffer. 1222 */ 1223 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1224 skb->len); 1225 ep->mpa_pkt_len += skb->len; 1226 1227 /* 1228 * if we don't even have the mpa message, then bail. 1229 */ 1230 if (ep->mpa_pkt_len < sizeof(*mpa)) 1231 return; 1232 mpa = (struct mpa_message *) ep->mpa_pkt; 1233 1234 /* Validate MPA header. */ 1235 if (mpa->revision > mpa_rev) { 1236 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1237 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1238 err = -EPROTO; 1239 goto err; 1240 } 1241 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1242 err = -EPROTO; 1243 goto err; 1244 } 1245 1246 plen = ntohs(mpa->private_data_size); 1247 1248 /* 1249 * Fail if there's too much private data. 1250 */ 1251 if (plen > MPA_MAX_PRIVATE_DATA) { 1252 err = -EPROTO; 1253 goto err; 1254 } 1255 1256 /* 1257 * If plen does not account for pkt size 1258 */ 1259 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1260 err = -EPROTO; 1261 goto err; 1262 } 1263 1264 ep->plen = (u8) plen; 1265 1266 /* 1267 * If we don't have all the pdata yet, then bail. 1268 * We'll continue process when more data arrives. 1269 */ 1270 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1271 return; 1272 1273 if (mpa->flags & MPA_REJECT) { 1274 err = -ECONNREFUSED; 1275 goto err; 1276 } 1277 1278 /* 1279 * If we get here we have accumulated the entire mpa 1280 * start reply message including private data. And 1281 * the MPA header is valid. 1282 */ 1283 state_set(&ep->com, FPDU_MODE); 1284 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1285 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1286 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1287 ep->mpa_attr.version = mpa->revision; 1288 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1289 1290 if (mpa->revision == 2) { 1291 ep->mpa_attr.enhanced_rdma_conn = 1292 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1293 if (ep->mpa_attr.enhanced_rdma_conn) { 1294 mpa_v2_params = (struct mpa_v2_conn_params *) 1295 (ep->mpa_pkt + sizeof(*mpa)); 1296 resp_ird = ntohs(mpa_v2_params->ird) & 1297 MPA_V2_IRD_ORD_MASK; 1298 resp_ord = ntohs(mpa_v2_params->ord) & 1299 MPA_V2_IRD_ORD_MASK; 1300 1301 /* 1302 * This is a double-check. Ideally, below checks are 1303 * not required since ird/ord stuff has been taken 1304 * care of in c4iw_accept_cr 1305 */ 1306 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1307 err = -ENOMEM; 1308 ep->ird = resp_ord; 1309 ep->ord = resp_ird; 1310 insuff_ird = 1; 1311 } 1312 1313 if (ntohs(mpa_v2_params->ird) & 1314 MPA_V2_PEER2PEER_MODEL) { 1315 if (ntohs(mpa_v2_params->ord) & 1316 MPA_V2_RDMA_WRITE_RTR) 1317 ep->mpa_attr.p2p_type = 1318 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1319 else if (ntohs(mpa_v2_params->ord) & 1320 MPA_V2_RDMA_READ_RTR) 1321 ep->mpa_attr.p2p_type = 1322 FW_RI_INIT_P2PTYPE_READ_REQ; 1323 } 1324 } 1325 } else if (mpa->revision == 1) 1326 if (peer2peer) 1327 ep->mpa_attr.p2p_type = p2p_type; 1328 1329 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1330 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1331 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1332 ep->mpa_attr.recv_marker_enabled, 1333 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1334 ep->mpa_attr.p2p_type, p2p_type); 1335 1336 /* 1337 * If responder's RTR does not match with that of initiator, assign 1338 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1339 * generated when moving QP to RTS state. 1340 * A TERM message will be sent after QP has moved to RTS state 1341 */ 1342 if ((ep->mpa_attr.version == 2) && peer2peer && 1343 (ep->mpa_attr.p2p_type != p2p_type)) { 1344 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1345 rtr_mismatch = 1; 1346 } 1347 1348 attrs.mpa_attr = ep->mpa_attr; 1349 attrs.max_ird = ep->ird; 1350 attrs.max_ord = ep->ord; 1351 attrs.llp_stream_handle = ep; 1352 attrs.next_state = C4IW_QP_STATE_RTS; 1353 1354 mask = C4IW_QP_ATTR_NEXT_STATE | 1355 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1356 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1357 1358 /* bind QP and TID with INIT_WR */ 1359 err = c4iw_modify_qp(ep->com.qp->rhp, 1360 ep->com.qp, mask, &attrs, 1); 1361 if (err) 1362 goto err; 1363 1364 /* 1365 * If responder's RTR requirement did not match with what initiator 1366 * supports, generate TERM message 1367 */ 1368 if (rtr_mismatch) { 1369 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1370 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1371 attrs.ecode = MPA_NOMATCH_RTR; 1372 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1373 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1374 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1375 err = -ENOMEM; 1376 goto out; 1377 } 1378 1379 /* 1380 * Generate TERM if initiator IRD is not sufficient for responder 1381 * provided ORD. Currently, we do the same behaviour even when 1382 * responder provided IRD is also not sufficient as regards to 1383 * initiator ORD. 1384 */ 1385 if (insuff_ird) { 1386 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1387 __func__); 1388 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1389 attrs.ecode = MPA_INSUFF_IRD; 1390 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1391 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1392 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1393 err = -ENOMEM; 1394 goto out; 1395 } 1396 goto out; 1397 err: 1398 state_set(&ep->com, ABORTING); 1399 send_abort(ep, skb, GFP_KERNEL); 1400 out: 1401 connect_reply_upcall(ep, err); 1402 return; 1403 } 1404 1405 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1406 { 1407 struct mpa_message *mpa; 1408 struct mpa_v2_conn_params *mpa_v2_params; 1409 u16 plen; 1410 1411 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1412 1413 if (state_read(&ep->com) != MPA_REQ_WAIT) 1414 return; 1415 1416 /* 1417 * If we get more than the supported amount of private data 1418 * then we must fail this connection. 1419 */ 1420 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1421 stop_ep_timer(ep); 1422 abort_connection(ep, skb, GFP_KERNEL); 1423 return; 1424 } 1425 1426 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1427 1428 /* 1429 * Copy the new data into our accumulation buffer. 1430 */ 1431 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1432 skb->len); 1433 ep->mpa_pkt_len += skb->len; 1434 1435 /* 1436 * If we don't even have the mpa message, then bail. 1437 * We'll continue process when more data arrives. 1438 */ 1439 if (ep->mpa_pkt_len < sizeof(*mpa)) 1440 return; 1441 1442 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1443 stop_ep_timer(ep); 1444 mpa = (struct mpa_message *) ep->mpa_pkt; 1445 1446 /* 1447 * Validate MPA Header. 1448 */ 1449 if (mpa->revision > mpa_rev) { 1450 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1451 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1452 stop_ep_timer(ep); 1453 abort_connection(ep, skb, GFP_KERNEL); 1454 return; 1455 } 1456 1457 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1458 stop_ep_timer(ep); 1459 abort_connection(ep, skb, GFP_KERNEL); 1460 return; 1461 } 1462 1463 plen = ntohs(mpa->private_data_size); 1464 1465 /* 1466 * Fail if there's too much private data. 1467 */ 1468 if (plen > MPA_MAX_PRIVATE_DATA) { 1469 stop_ep_timer(ep); 1470 abort_connection(ep, skb, GFP_KERNEL); 1471 return; 1472 } 1473 1474 /* 1475 * If plen does not account for pkt size 1476 */ 1477 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1478 stop_ep_timer(ep); 1479 abort_connection(ep, skb, GFP_KERNEL); 1480 return; 1481 } 1482 ep->plen = (u8) plen; 1483 1484 /* 1485 * If we don't have all the pdata yet, then bail. 1486 */ 1487 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1488 return; 1489 1490 /* 1491 * If we get here we have accumulated the entire mpa 1492 * start reply message including private data. 1493 */ 1494 ep->mpa_attr.initiator = 0; 1495 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1496 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1497 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1498 ep->mpa_attr.version = mpa->revision; 1499 if (mpa->revision == 1) 1500 ep->tried_with_mpa_v1 = 1; 1501 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1502 1503 if (mpa->revision == 2) { 1504 ep->mpa_attr.enhanced_rdma_conn = 1505 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1506 if (ep->mpa_attr.enhanced_rdma_conn) { 1507 mpa_v2_params = (struct mpa_v2_conn_params *) 1508 (ep->mpa_pkt + sizeof(*mpa)); 1509 ep->ird = ntohs(mpa_v2_params->ird) & 1510 MPA_V2_IRD_ORD_MASK; 1511 ep->ord = ntohs(mpa_v2_params->ord) & 1512 MPA_V2_IRD_ORD_MASK; 1513 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1514 if (peer2peer) { 1515 if (ntohs(mpa_v2_params->ord) & 1516 MPA_V2_RDMA_WRITE_RTR) 1517 ep->mpa_attr.p2p_type = 1518 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1519 else if (ntohs(mpa_v2_params->ord) & 1520 MPA_V2_RDMA_READ_RTR) 1521 ep->mpa_attr.p2p_type = 1522 FW_RI_INIT_P2PTYPE_READ_REQ; 1523 } 1524 } 1525 } else if (mpa->revision == 1) 1526 if (peer2peer) 1527 ep->mpa_attr.p2p_type = p2p_type; 1528 1529 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1530 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1531 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1532 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1533 ep->mpa_attr.p2p_type); 1534 1535 state_set(&ep->com, MPA_REQ_RCVD); 1536 1537 /* drive upcall */ 1538 connect_request_upcall(ep); 1539 return; 1540 } 1541 1542 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1543 { 1544 struct c4iw_ep *ep; 1545 struct cpl_rx_data *hdr = cplhdr(skb); 1546 unsigned int dlen = ntohs(hdr->len); 1547 unsigned int tid = GET_TID(hdr); 1548 struct tid_info *t = dev->rdev.lldi.tids; 1549 __u8 status = hdr->status; 1550 1551 ep = lookup_tid(t, tid); 1552 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1553 skb_pull(skb, sizeof(*hdr)); 1554 skb_trim(skb, dlen); 1555 1556 /* update RX credits */ 1557 update_rx_credits(ep, dlen); 1558 1559 switch (state_read(&ep->com)) { 1560 case MPA_REQ_SENT: 1561 ep->rcv_seq += dlen; 1562 process_mpa_reply(ep, skb); 1563 break; 1564 case MPA_REQ_WAIT: 1565 ep->rcv_seq += dlen; 1566 process_mpa_request(ep, skb); 1567 break; 1568 case FPDU_MODE: { 1569 struct c4iw_qp_attributes attrs; 1570 BUG_ON(!ep->com.qp); 1571 if (status) 1572 pr_err("%s Unexpected streaming data." \ 1573 " qpid %u ep %p state %d tid %u status %d\n", 1574 __func__, ep->com.qp->wq.sq.qid, ep, 1575 state_read(&ep->com), ep->hwtid, status); 1576 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1577 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1578 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1579 break; 1580 } 1581 default: 1582 break; 1583 } 1584 return 0; 1585 } 1586 1587 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1588 { 1589 struct c4iw_ep *ep; 1590 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1591 int release = 0; 1592 unsigned int tid = GET_TID(rpl); 1593 struct tid_info *t = dev->rdev.lldi.tids; 1594 1595 ep = lookup_tid(t, tid); 1596 if (!ep) { 1597 printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n"); 1598 return 0; 1599 } 1600 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1601 mutex_lock(&ep->com.mutex); 1602 switch (ep->com.state) { 1603 case ABORTING: 1604 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1605 __state_set(&ep->com, DEAD); 1606 release = 1; 1607 break; 1608 default: 1609 printk(KERN_ERR "%s ep %p state %d\n", 1610 __func__, ep, ep->com.state); 1611 break; 1612 } 1613 mutex_unlock(&ep->com.mutex); 1614 1615 if (release) 1616 release_ep_resources(ep); 1617 return 0; 1618 } 1619 1620 static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid) 1621 { 1622 struct sk_buff *skb; 1623 struct fw_ofld_connection_wr *req; 1624 unsigned int mtu_idx; 1625 int wscale; 1626 struct sockaddr_in *sin; 1627 1628 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1629 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1630 memset(req, 0, sizeof(*req)); 1631 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR)); 1632 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1633 req->le.filter = cpu_to_be32(select_ntuple(ep->com.dev, ep->dst, 1634 ep->l2t)); 1635 sin = (struct sockaddr_in *)&ep->com.local_addr; 1636 req->le.lport = sin->sin_port; 1637 req->le.u.ipv4.lip = sin->sin_addr.s_addr; 1638 sin = (struct sockaddr_in *)&ep->com.remote_addr; 1639 req->le.pport = sin->sin_port; 1640 req->le.u.ipv4.pip = sin->sin_addr.s_addr; 1641 req->tcb.t_state_to_astid = 1642 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1643 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1644 req->tcb.cplrxdataack_cplpassacceptrpl = 1645 htons(F_FW_OFLD_CONNECTION_WR_CPLRXDATAACK); 1646 req->tcb.tx_max = (__force __be32) jiffies; 1647 req->tcb.rcv_adv = htons(1); 1648 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1649 wscale = compute_wscale(rcv_win); 1650 req->tcb.opt0 = (__force __be64) (TCAM_BYPASS(1) | 1651 (nocong ? NO_CONG(1) : 0) | 1652 KEEP_ALIVE(1) | 1653 DELACK(1) | 1654 WND_SCALE(wscale) | 1655 MSS_IDX(mtu_idx) | 1656 L2T_IDX(ep->l2t->idx) | 1657 TX_CHAN(ep->tx_chan) | 1658 SMAC_SEL(ep->smac_idx) | 1659 DSCP(ep->tos) | 1660 ULP_MODE(ULP_MODE_TCPDDP) | 1661 RCV_BUFSIZ(rcv_win >> 10)); 1662 req->tcb.opt2 = (__force __be32) (PACE(1) | 1663 TX_QUEUE(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) | 1664 RX_CHANNEL(0) | 1665 CCTRL_ECN(enable_ecn) | 1666 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid)); 1667 if (enable_tcp_timestamps) 1668 req->tcb.opt2 |= (__force __be32) TSTAMPS_EN(1); 1669 if (enable_tcp_sack) 1670 req->tcb.opt2 |= (__force __be32) SACK_EN(1); 1671 if (wscale && enable_tcp_window_scaling) 1672 req->tcb.opt2 |= (__force __be32) WND_SCALE_EN(1); 1673 req->tcb.opt0 = cpu_to_be64((__force u64) req->tcb.opt0); 1674 req->tcb.opt2 = cpu_to_be32((__force u32) req->tcb.opt2); 1675 set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx); 1676 set_bit(ACT_OFLD_CONN, &ep->com.history); 1677 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1678 } 1679 1680 /* 1681 * Return whether a failed active open has allocated a TID 1682 */ 1683 static inline int act_open_has_tid(int status) 1684 { 1685 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1686 status != CPL_ERR_ARP_MISS; 1687 } 1688 1689 #define ACT_OPEN_RETRY_COUNT 2 1690 1691 static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip, 1692 struct dst_entry *dst, struct c4iw_dev *cdev, 1693 bool clear_mpa_v1) 1694 { 1695 struct neighbour *n; 1696 int err, step; 1697 struct net_device *pdev; 1698 1699 n = dst_neigh_lookup(dst, peer_ip); 1700 if (!n) 1701 return -ENODEV; 1702 1703 rcu_read_lock(); 1704 err = -ENOMEM; 1705 if (n->dev->flags & IFF_LOOPBACK) { 1706 if (iptype == 4) 1707 pdev = ip_dev_find(&init_net, *(__be32 *)peer_ip); 1708 else if (IS_ENABLED(CONFIG_IPV6)) 1709 for_each_netdev(&init_net, pdev) { 1710 if (ipv6_chk_addr(&init_net, 1711 (struct in6_addr *)peer_ip, 1712 pdev, 1)) 1713 break; 1714 } 1715 else 1716 pdev = NULL; 1717 1718 if (!pdev) { 1719 err = -ENODEV; 1720 goto out; 1721 } 1722 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1723 n, pdev, 0); 1724 if (!ep->l2t) 1725 goto out; 1726 ep->mtu = pdev->mtu; 1727 ep->tx_chan = cxgb4_port_chan(pdev); 1728 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1729 step = cdev->rdev.lldi.ntxq / 1730 cdev->rdev.lldi.nchan; 1731 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1732 step = cdev->rdev.lldi.nrxq / 1733 cdev->rdev.lldi.nchan; 1734 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1735 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1736 cxgb4_port_idx(pdev) * step]; 1737 dev_put(pdev); 1738 } else { 1739 pdev = get_real_dev(n->dev); 1740 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1741 n, pdev, 0); 1742 if (!ep->l2t) 1743 goto out; 1744 ep->mtu = dst_mtu(dst); 1745 ep->tx_chan = cxgb4_port_chan(n->dev); 1746 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1747 step = cdev->rdev.lldi.ntxq / 1748 cdev->rdev.lldi.nchan; 1749 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1750 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1751 step = cdev->rdev.lldi.nrxq / 1752 cdev->rdev.lldi.nchan; 1753 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1754 cxgb4_port_idx(n->dev) * step]; 1755 1756 if (clear_mpa_v1) { 1757 ep->retry_with_mpa_v1 = 0; 1758 ep->tried_with_mpa_v1 = 0; 1759 } 1760 } 1761 err = 0; 1762 out: 1763 rcu_read_unlock(); 1764 1765 neigh_release(n); 1766 1767 return err; 1768 } 1769 1770 static int c4iw_reconnect(struct c4iw_ep *ep) 1771 { 1772 int err = 0; 1773 struct sockaddr_in *laddr = (struct sockaddr_in *) 1774 &ep->com.cm_id->local_addr; 1775 struct sockaddr_in *raddr = (struct sockaddr_in *) 1776 &ep->com.cm_id->remote_addr; 1777 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *) 1778 &ep->com.cm_id->local_addr; 1779 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 1780 &ep->com.cm_id->remote_addr; 1781 int iptype; 1782 __u8 *ra; 1783 1784 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1785 init_timer(&ep->timer); 1786 1787 /* 1788 * Allocate an active TID to initiate a TCP connection. 1789 */ 1790 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1791 if (ep->atid == -1) { 1792 pr_err("%s - cannot alloc atid.\n", __func__); 1793 err = -ENOMEM; 1794 goto fail2; 1795 } 1796 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1797 1798 /* find a route */ 1799 if (ep->com.cm_id->local_addr.ss_family == AF_INET) { 1800 ep->dst = find_route(ep->com.dev, laddr->sin_addr.s_addr, 1801 raddr->sin_addr.s_addr, laddr->sin_port, 1802 raddr->sin_port, 0); 1803 iptype = 4; 1804 ra = (__u8 *)&raddr->sin_addr; 1805 } else { 1806 ep->dst = find_route6(ep->com.dev, laddr6->sin6_addr.s6_addr, 1807 raddr6->sin6_addr.s6_addr, 1808 laddr6->sin6_port, raddr6->sin6_port, 0, 1809 raddr6->sin6_scope_id); 1810 iptype = 6; 1811 ra = (__u8 *)&raddr6->sin6_addr; 1812 } 1813 if (!ep->dst) { 1814 pr_err("%s - cannot find route.\n", __func__); 1815 err = -EHOSTUNREACH; 1816 goto fail3; 1817 } 1818 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, false); 1819 if (err) { 1820 pr_err("%s - cannot alloc l2e.\n", __func__); 1821 goto fail4; 1822 } 1823 1824 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1825 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1826 ep->l2t->idx); 1827 1828 state_set(&ep->com, CONNECTING); 1829 ep->tos = 0; 1830 1831 /* send connect request to rnic */ 1832 err = send_connect(ep); 1833 if (!err) 1834 goto out; 1835 1836 cxgb4_l2t_release(ep->l2t); 1837 fail4: 1838 dst_release(ep->dst); 1839 fail3: 1840 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 1841 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1842 fail2: 1843 /* 1844 * remember to send notification to upper layer. 1845 * We are in here so the upper layer is not aware that this is 1846 * re-connect attempt and so, upper layer is still waiting for 1847 * response of 1st connect request. 1848 */ 1849 connect_reply_upcall(ep, -ECONNRESET); 1850 c4iw_put_ep(&ep->com); 1851 out: 1852 return err; 1853 } 1854 1855 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1856 { 1857 struct c4iw_ep *ep; 1858 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1859 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1860 ntohl(rpl->atid_status))); 1861 struct tid_info *t = dev->rdev.lldi.tids; 1862 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1863 struct sockaddr_in *la; 1864 struct sockaddr_in *ra; 1865 struct sockaddr_in6 *la6; 1866 struct sockaddr_in6 *ra6; 1867 1868 ep = lookup_atid(t, atid); 1869 la = (struct sockaddr_in *)&ep->com.local_addr; 1870 ra = (struct sockaddr_in *)&ep->com.remote_addr; 1871 la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 1872 ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 1873 1874 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1875 status, status2errno(status)); 1876 1877 if (status == CPL_ERR_RTX_NEG_ADVICE) { 1878 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1879 atid); 1880 return 0; 1881 } 1882 1883 set_bit(ACT_OPEN_RPL, &ep->com.history); 1884 1885 /* 1886 * Log interesting failures. 1887 */ 1888 switch (status) { 1889 case CPL_ERR_CONN_RESET: 1890 case CPL_ERR_CONN_TIMEDOUT: 1891 break; 1892 case CPL_ERR_TCAM_FULL: 1893 mutex_lock(&dev->rdev.stats.lock); 1894 dev->rdev.stats.tcam_full++; 1895 mutex_unlock(&dev->rdev.stats.lock); 1896 if (ep->com.local_addr.ss_family == AF_INET && 1897 dev->rdev.lldi.enable_fw_ofld_conn) { 1898 send_fw_act_open_req(ep, 1899 GET_TID_TID(GET_AOPEN_ATID( 1900 ntohl(rpl->atid_status)))); 1901 return 0; 1902 } 1903 break; 1904 case CPL_ERR_CONN_EXIST: 1905 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 1906 set_bit(ACT_RETRY_INUSE, &ep->com.history); 1907 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, 1908 atid); 1909 cxgb4_free_atid(t, atid); 1910 dst_release(ep->dst); 1911 cxgb4_l2t_release(ep->l2t); 1912 c4iw_reconnect(ep); 1913 return 0; 1914 } 1915 break; 1916 default: 1917 if (ep->com.local_addr.ss_family == AF_INET) { 1918 pr_info("Active open failure - atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1919 atid, status, status2errno(status), 1920 &la->sin_addr.s_addr, ntohs(la->sin_port), 1921 &ra->sin_addr.s_addr, ntohs(ra->sin_port)); 1922 } else { 1923 pr_info("Active open failure - atid %u status %u errno %d %pI6:%u->%pI6:%u\n", 1924 atid, status, status2errno(status), 1925 la6->sin6_addr.s6_addr, ntohs(la6->sin6_port), 1926 ra6->sin6_addr.s6_addr, ntohs(ra6->sin6_port)); 1927 } 1928 break; 1929 } 1930 1931 connect_reply_upcall(ep, status2errno(status)); 1932 state_set(&ep->com, DEAD); 1933 1934 if (status && act_open_has_tid(status)) 1935 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1936 1937 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 1938 cxgb4_free_atid(t, atid); 1939 dst_release(ep->dst); 1940 cxgb4_l2t_release(ep->l2t); 1941 c4iw_put_ep(&ep->com); 1942 1943 return 0; 1944 } 1945 1946 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1947 { 1948 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1949 struct tid_info *t = dev->rdev.lldi.tids; 1950 unsigned int stid = GET_TID(rpl); 1951 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1952 1953 if (!ep) { 1954 PDBG("%s stid %d lookup failure!\n", __func__, stid); 1955 goto out; 1956 } 1957 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1958 rpl->status, status2errno(rpl->status)); 1959 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1960 1961 out: 1962 return 0; 1963 } 1964 1965 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1966 { 1967 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1968 struct tid_info *t = dev->rdev.lldi.tids; 1969 unsigned int stid = GET_TID(rpl); 1970 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1971 1972 PDBG("%s ep %p\n", __func__, ep); 1973 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1974 return 0; 1975 } 1976 1977 static void accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, 1978 struct cpl_pass_accept_req *req) 1979 { 1980 struct cpl_pass_accept_rpl *rpl; 1981 unsigned int mtu_idx; 1982 u64 opt0; 1983 u32 opt2; 1984 int wscale; 1985 1986 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1987 BUG_ON(skb_cloned(skb)); 1988 skb_trim(skb, sizeof(*rpl)); 1989 skb_get(skb); 1990 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1991 wscale = compute_wscale(rcv_win); 1992 opt0 = (nocong ? NO_CONG(1) : 0) | 1993 KEEP_ALIVE(1) | 1994 DELACK(1) | 1995 WND_SCALE(wscale) | 1996 MSS_IDX(mtu_idx) | 1997 L2T_IDX(ep->l2t->idx) | 1998 TX_CHAN(ep->tx_chan) | 1999 SMAC_SEL(ep->smac_idx) | 2000 DSCP(ep->tos >> 2) | 2001 ULP_MODE(ULP_MODE_TCPDDP) | 2002 RCV_BUFSIZ(rcv_win>>10); 2003 opt2 = RX_CHANNEL(0) | 2004 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 2005 2006 if (enable_tcp_timestamps && req->tcpopt.tstamp) 2007 opt2 |= TSTAMPS_EN(1); 2008 if (enable_tcp_sack && req->tcpopt.sack) 2009 opt2 |= SACK_EN(1); 2010 if (wscale && enable_tcp_window_scaling) 2011 opt2 |= WND_SCALE_EN(1); 2012 if (enable_ecn) { 2013 const struct tcphdr *tcph; 2014 u32 hlen = ntohl(req->hdr_len); 2015 2016 tcph = (const void *)(req + 1) + G_ETH_HDR_LEN(hlen) + 2017 G_IP_HDR_LEN(hlen); 2018 if (tcph->ece && tcph->cwr) 2019 opt2 |= CCTRL_ECN(1); 2020 } 2021 2022 rpl = cplhdr(skb); 2023 INIT_TP_WR(rpl, ep->hwtid); 2024 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 2025 ep->hwtid)); 2026 rpl->opt0 = cpu_to_be64(opt0); 2027 rpl->opt2 = cpu_to_be32(opt2); 2028 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 2029 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 2030 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 2031 2032 return; 2033 } 2034 2035 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, struct sk_buff *skb) 2036 { 2037 PDBG("%s c4iw_dev %p tid %u\n", __func__, dev, hwtid); 2038 BUG_ON(skb_cloned(skb)); 2039 skb_trim(skb, sizeof(struct cpl_tid_release)); 2040 skb_get(skb); 2041 release_tid(&dev->rdev, hwtid, skb); 2042 return; 2043 } 2044 2045 static void get_4tuple(struct cpl_pass_accept_req *req, int *iptype, 2046 __u8 *local_ip, __u8 *peer_ip, 2047 __be16 *local_port, __be16 *peer_port) 2048 { 2049 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 2050 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 2051 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 2052 struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len); 2053 struct tcphdr *tcp = (struct tcphdr *) 2054 ((u8 *)(req + 1) + eth_len + ip_len); 2055 2056 if (ip->version == 4) { 2057 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 2058 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 2059 ntohs(tcp->dest)); 2060 *iptype = 4; 2061 memcpy(peer_ip, &ip->saddr, 4); 2062 memcpy(local_ip, &ip->daddr, 4); 2063 } else { 2064 PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__, 2065 ip6->saddr.s6_addr, ip6->daddr.s6_addr, ntohs(tcp->source), 2066 ntohs(tcp->dest)); 2067 *iptype = 6; 2068 memcpy(peer_ip, ip6->saddr.s6_addr, 16); 2069 memcpy(local_ip, ip6->daddr.s6_addr, 16); 2070 } 2071 *peer_port = tcp->source; 2072 *local_port = tcp->dest; 2073 2074 return; 2075 } 2076 2077 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 2078 { 2079 struct c4iw_ep *child_ep = NULL, *parent_ep; 2080 struct cpl_pass_accept_req *req = cplhdr(skb); 2081 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 2082 struct tid_info *t = dev->rdev.lldi.tids; 2083 unsigned int hwtid = GET_TID(req); 2084 struct dst_entry *dst; 2085 __u8 local_ip[16], peer_ip[16]; 2086 __be16 local_port, peer_port; 2087 int err; 2088 u16 peer_mss = ntohs(req->tcpopt.mss); 2089 int iptype; 2090 2091 parent_ep = lookup_stid(t, stid); 2092 if (!parent_ep) { 2093 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 2094 goto reject; 2095 } 2096 2097 if (state_read(&parent_ep->com) != LISTEN) { 2098 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 2099 __func__); 2100 goto reject; 2101 } 2102 2103 get_4tuple(req, &iptype, local_ip, peer_ip, &local_port, &peer_port); 2104 2105 /* Find output route */ 2106 if (iptype == 4) { 2107 PDBG("%s parent ep %p hwtid %u laddr %pI4 raddr %pI4 lport %d rport %d peer_mss %d\n" 2108 , __func__, parent_ep, hwtid, 2109 local_ip, peer_ip, ntohs(local_port), 2110 ntohs(peer_port), peer_mss); 2111 dst = find_route(dev, *(__be32 *)local_ip, *(__be32 *)peer_ip, 2112 local_port, peer_port, 2113 GET_POPEN_TOS(ntohl(req->tos_stid))); 2114 } else { 2115 PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n" 2116 , __func__, parent_ep, hwtid, 2117 local_ip, peer_ip, ntohs(local_port), 2118 ntohs(peer_port), peer_mss); 2119 dst = find_route6(dev, local_ip, peer_ip, local_port, peer_port, 2120 PASS_OPEN_TOS(ntohl(req->tos_stid)), 2121 ((struct sockaddr_in6 *) 2122 &parent_ep->com.local_addr)->sin6_scope_id); 2123 } 2124 if (!dst) { 2125 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 2126 __func__); 2127 goto reject; 2128 } 2129 2130 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 2131 if (!child_ep) { 2132 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 2133 __func__); 2134 dst_release(dst); 2135 goto reject; 2136 } 2137 2138 err = import_ep(child_ep, iptype, peer_ip, dst, dev, false); 2139 if (err) { 2140 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 2141 __func__); 2142 dst_release(dst); 2143 kfree(child_ep); 2144 goto reject; 2145 } 2146 2147 if (peer_mss && child_ep->mtu > (peer_mss + 40)) 2148 child_ep->mtu = peer_mss + 40; 2149 2150 state_set(&child_ep->com, CONNECTING); 2151 child_ep->com.dev = dev; 2152 child_ep->com.cm_id = NULL; 2153 if (iptype == 4) { 2154 struct sockaddr_in *sin = (struct sockaddr_in *) 2155 &child_ep->com.local_addr; 2156 sin->sin_family = PF_INET; 2157 sin->sin_port = local_port; 2158 sin->sin_addr.s_addr = *(__be32 *)local_ip; 2159 sin = (struct sockaddr_in *)&child_ep->com.remote_addr; 2160 sin->sin_family = PF_INET; 2161 sin->sin_port = peer_port; 2162 sin->sin_addr.s_addr = *(__be32 *)peer_ip; 2163 } else { 2164 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 2165 &child_ep->com.local_addr; 2166 sin6->sin6_family = PF_INET6; 2167 sin6->sin6_port = local_port; 2168 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16); 2169 sin6 = (struct sockaddr_in6 *)&child_ep->com.remote_addr; 2170 sin6->sin6_family = PF_INET6; 2171 sin6->sin6_port = peer_port; 2172 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16); 2173 } 2174 c4iw_get_ep(&parent_ep->com); 2175 child_ep->parent_ep = parent_ep; 2176 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 2177 child_ep->dst = dst; 2178 child_ep->hwtid = hwtid; 2179 2180 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 2181 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 2182 2183 init_timer(&child_ep->timer); 2184 cxgb4_insert_tid(t, child_ep, hwtid); 2185 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid); 2186 accept_cr(child_ep, skb, req); 2187 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 2188 goto out; 2189 reject: 2190 reject_cr(dev, hwtid, skb); 2191 out: 2192 return 0; 2193 } 2194 2195 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 2196 { 2197 struct c4iw_ep *ep; 2198 struct cpl_pass_establish *req = cplhdr(skb); 2199 struct tid_info *t = dev->rdev.lldi.tids; 2200 unsigned int tid = GET_TID(req); 2201 2202 ep = lookup_tid(t, tid); 2203 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2204 ep->snd_seq = be32_to_cpu(req->snd_isn); 2205 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 2206 2207 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__, ep, tid, 2208 ntohs(req->tcp_opt)); 2209 2210 set_emss(ep, ntohs(req->tcp_opt)); 2211 2212 dst_confirm(ep->dst); 2213 state_set(&ep->com, MPA_REQ_WAIT); 2214 start_ep_timer(ep); 2215 send_flowc(ep, skb); 2216 set_bit(PASS_ESTAB, &ep->com.history); 2217 2218 return 0; 2219 } 2220 2221 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 2222 { 2223 struct cpl_peer_close *hdr = cplhdr(skb); 2224 struct c4iw_ep *ep; 2225 struct c4iw_qp_attributes attrs; 2226 int disconnect = 1; 2227 int release = 0; 2228 struct tid_info *t = dev->rdev.lldi.tids; 2229 unsigned int tid = GET_TID(hdr); 2230 int ret; 2231 2232 ep = lookup_tid(t, tid); 2233 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2234 dst_confirm(ep->dst); 2235 2236 set_bit(PEER_CLOSE, &ep->com.history); 2237 mutex_lock(&ep->com.mutex); 2238 switch (ep->com.state) { 2239 case MPA_REQ_WAIT: 2240 __state_set(&ep->com, CLOSING); 2241 break; 2242 case MPA_REQ_SENT: 2243 __state_set(&ep->com, CLOSING); 2244 connect_reply_upcall(ep, -ECONNRESET); 2245 break; 2246 case MPA_REQ_RCVD: 2247 2248 /* 2249 * We're gonna mark this puppy DEAD, but keep 2250 * the reference on it until the ULP accepts or 2251 * rejects the CR. Also wake up anyone waiting 2252 * in rdma connection migration (see c4iw_accept_cr()). 2253 */ 2254 __state_set(&ep->com, CLOSING); 2255 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2256 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2257 break; 2258 case MPA_REP_SENT: 2259 __state_set(&ep->com, CLOSING); 2260 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2261 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2262 break; 2263 case FPDU_MODE: 2264 start_ep_timer(ep); 2265 __state_set(&ep->com, CLOSING); 2266 attrs.next_state = C4IW_QP_STATE_CLOSING; 2267 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2268 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2269 if (ret != -ECONNRESET) { 2270 peer_close_upcall(ep); 2271 disconnect = 1; 2272 } 2273 break; 2274 case ABORTING: 2275 disconnect = 0; 2276 break; 2277 case CLOSING: 2278 __state_set(&ep->com, MORIBUND); 2279 disconnect = 0; 2280 break; 2281 case MORIBUND: 2282 stop_ep_timer(ep); 2283 if (ep->com.cm_id && ep->com.qp) { 2284 attrs.next_state = C4IW_QP_STATE_IDLE; 2285 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2286 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2287 } 2288 close_complete_upcall(ep); 2289 __state_set(&ep->com, DEAD); 2290 release = 1; 2291 disconnect = 0; 2292 break; 2293 case DEAD: 2294 disconnect = 0; 2295 break; 2296 default: 2297 BUG_ON(1); 2298 } 2299 mutex_unlock(&ep->com.mutex); 2300 if (disconnect) 2301 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2302 if (release) 2303 release_ep_resources(ep); 2304 return 0; 2305 } 2306 2307 /* 2308 * Returns whether an ABORT_REQ_RSS message is a negative advice. 2309 */ 2310 static int is_neg_adv_abort(unsigned int status) 2311 { 2312 return status == CPL_ERR_RTX_NEG_ADVICE || 2313 status == CPL_ERR_PERSIST_NEG_ADVICE; 2314 } 2315 2316 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 2317 { 2318 struct cpl_abort_req_rss *req = cplhdr(skb); 2319 struct c4iw_ep *ep; 2320 struct cpl_abort_rpl *rpl; 2321 struct sk_buff *rpl_skb; 2322 struct c4iw_qp_attributes attrs; 2323 int ret; 2324 int release = 0; 2325 struct tid_info *t = dev->rdev.lldi.tids; 2326 unsigned int tid = GET_TID(req); 2327 2328 ep = lookup_tid(t, tid); 2329 if (is_neg_adv_abort(req->status)) { 2330 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2331 ep->hwtid); 2332 return 0; 2333 } 2334 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2335 ep->com.state); 2336 set_bit(PEER_ABORT, &ep->com.history); 2337 2338 /* 2339 * Wake up any threads in rdma_init() or rdma_fini(). 2340 * However, this is not needed if com state is just 2341 * MPA_REQ_SENT 2342 */ 2343 if (ep->com.state != MPA_REQ_SENT) 2344 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2345 2346 mutex_lock(&ep->com.mutex); 2347 switch (ep->com.state) { 2348 case CONNECTING: 2349 break; 2350 case MPA_REQ_WAIT: 2351 stop_ep_timer(ep); 2352 break; 2353 case MPA_REQ_SENT: 2354 stop_ep_timer(ep); 2355 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1)) 2356 connect_reply_upcall(ep, -ECONNRESET); 2357 else { 2358 /* 2359 * we just don't send notification upwards because we 2360 * want to retry with mpa_v1 without upper layers even 2361 * knowing it. 2362 * 2363 * do some housekeeping so as to re-initiate the 2364 * connection 2365 */ 2366 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 2367 mpa_rev); 2368 ep->retry_with_mpa_v1 = 1; 2369 } 2370 break; 2371 case MPA_REP_SENT: 2372 break; 2373 case MPA_REQ_RCVD: 2374 break; 2375 case MORIBUND: 2376 case CLOSING: 2377 stop_ep_timer(ep); 2378 /*FALLTHROUGH*/ 2379 case FPDU_MODE: 2380 if (ep->com.cm_id && ep->com.qp) { 2381 attrs.next_state = C4IW_QP_STATE_ERROR; 2382 ret = c4iw_modify_qp(ep->com.qp->rhp, 2383 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2384 &attrs, 1); 2385 if (ret) 2386 printk(KERN_ERR MOD 2387 "%s - qp <- error failed!\n", 2388 __func__); 2389 } 2390 peer_abort_upcall(ep); 2391 break; 2392 case ABORTING: 2393 break; 2394 case DEAD: 2395 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 2396 mutex_unlock(&ep->com.mutex); 2397 return 0; 2398 default: 2399 BUG_ON(1); 2400 break; 2401 } 2402 dst_confirm(ep->dst); 2403 if (ep->com.state != ABORTING) { 2404 __state_set(&ep->com, DEAD); 2405 /* we don't release if we want to retry with mpa_v1 */ 2406 if (!ep->retry_with_mpa_v1) 2407 release = 1; 2408 } 2409 mutex_unlock(&ep->com.mutex); 2410 2411 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 2412 if (!rpl_skb) { 2413 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 2414 __func__); 2415 release = 1; 2416 goto out; 2417 } 2418 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 2419 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 2420 INIT_TP_WR(rpl, ep->hwtid); 2421 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 2422 rpl->cmd = CPL_ABORT_NO_RST; 2423 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 2424 out: 2425 if (release) 2426 release_ep_resources(ep); 2427 else if (ep->retry_with_mpa_v1) { 2428 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 2429 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 2430 dst_release(ep->dst); 2431 cxgb4_l2t_release(ep->l2t); 2432 c4iw_reconnect(ep); 2433 } 2434 2435 return 0; 2436 } 2437 2438 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2439 { 2440 struct c4iw_ep *ep; 2441 struct c4iw_qp_attributes attrs; 2442 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2443 int release = 0; 2444 struct tid_info *t = dev->rdev.lldi.tids; 2445 unsigned int tid = GET_TID(rpl); 2446 2447 ep = lookup_tid(t, tid); 2448 2449 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2450 BUG_ON(!ep); 2451 2452 /* The cm_id may be null if we failed to connect */ 2453 mutex_lock(&ep->com.mutex); 2454 switch (ep->com.state) { 2455 case CLOSING: 2456 __state_set(&ep->com, MORIBUND); 2457 break; 2458 case MORIBUND: 2459 stop_ep_timer(ep); 2460 if ((ep->com.cm_id) && (ep->com.qp)) { 2461 attrs.next_state = C4IW_QP_STATE_IDLE; 2462 c4iw_modify_qp(ep->com.qp->rhp, 2463 ep->com.qp, 2464 C4IW_QP_ATTR_NEXT_STATE, 2465 &attrs, 1); 2466 } 2467 close_complete_upcall(ep); 2468 __state_set(&ep->com, DEAD); 2469 release = 1; 2470 break; 2471 case ABORTING: 2472 case DEAD: 2473 break; 2474 default: 2475 BUG_ON(1); 2476 break; 2477 } 2478 mutex_unlock(&ep->com.mutex); 2479 if (release) 2480 release_ep_resources(ep); 2481 return 0; 2482 } 2483 2484 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2485 { 2486 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2487 struct tid_info *t = dev->rdev.lldi.tids; 2488 unsigned int tid = GET_TID(rpl); 2489 struct c4iw_ep *ep; 2490 struct c4iw_qp_attributes attrs; 2491 2492 ep = lookup_tid(t, tid); 2493 BUG_ON(!ep); 2494 2495 if (ep && ep->com.qp) { 2496 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2497 ep->com.qp->wq.sq.qid); 2498 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2499 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2500 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2501 } else 2502 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2503 2504 return 0; 2505 } 2506 2507 /* 2508 * Upcall from the adapter indicating data has been transmitted. 2509 * For us its just the single MPA request or reply. We can now free 2510 * the skb holding the mpa message. 2511 */ 2512 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2513 { 2514 struct c4iw_ep *ep; 2515 struct cpl_fw4_ack *hdr = cplhdr(skb); 2516 u8 credits = hdr->credits; 2517 unsigned int tid = GET_TID(hdr); 2518 struct tid_info *t = dev->rdev.lldi.tids; 2519 2520 2521 ep = lookup_tid(t, tid); 2522 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2523 if (credits == 0) { 2524 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2525 __func__, ep, ep->hwtid, state_read(&ep->com)); 2526 return 0; 2527 } 2528 2529 dst_confirm(ep->dst); 2530 if (ep->mpa_skb) { 2531 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2532 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2533 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2534 kfree_skb(ep->mpa_skb); 2535 ep->mpa_skb = NULL; 2536 } 2537 return 0; 2538 } 2539 2540 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2541 { 2542 int err; 2543 struct c4iw_ep *ep = to_ep(cm_id); 2544 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2545 2546 if (state_read(&ep->com) == DEAD) { 2547 c4iw_put_ep(&ep->com); 2548 return -ECONNRESET; 2549 } 2550 set_bit(ULP_REJECT, &ep->com.history); 2551 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2552 if (mpa_rev == 0) 2553 abort_connection(ep, NULL, GFP_KERNEL); 2554 else { 2555 err = send_mpa_reject(ep, pdata, pdata_len); 2556 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2557 } 2558 c4iw_put_ep(&ep->com); 2559 return 0; 2560 } 2561 2562 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2563 { 2564 int err; 2565 struct c4iw_qp_attributes attrs; 2566 enum c4iw_qp_attr_mask mask; 2567 struct c4iw_ep *ep = to_ep(cm_id); 2568 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2569 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2570 2571 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2572 if (state_read(&ep->com) == DEAD) { 2573 err = -ECONNRESET; 2574 goto err; 2575 } 2576 2577 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2578 BUG_ON(!qp); 2579 2580 set_bit(ULP_ACCEPT, &ep->com.history); 2581 if ((conn_param->ord > c4iw_max_read_depth) || 2582 (conn_param->ird > c4iw_max_read_depth)) { 2583 abort_connection(ep, NULL, GFP_KERNEL); 2584 err = -EINVAL; 2585 goto err; 2586 } 2587 2588 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2589 if (conn_param->ord > ep->ird) { 2590 ep->ird = conn_param->ird; 2591 ep->ord = conn_param->ord; 2592 send_mpa_reject(ep, conn_param->private_data, 2593 conn_param->private_data_len); 2594 abort_connection(ep, NULL, GFP_KERNEL); 2595 err = -ENOMEM; 2596 goto err; 2597 } 2598 if (conn_param->ird > ep->ord) { 2599 if (!ep->ord) 2600 conn_param->ird = 1; 2601 else { 2602 abort_connection(ep, NULL, GFP_KERNEL); 2603 err = -ENOMEM; 2604 goto err; 2605 } 2606 } 2607 2608 } 2609 ep->ird = conn_param->ird; 2610 ep->ord = conn_param->ord; 2611 2612 if (ep->mpa_attr.version != 2) 2613 if (peer2peer && ep->ird == 0) 2614 ep->ird = 1; 2615 2616 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2617 2618 cm_id->add_ref(cm_id); 2619 ep->com.cm_id = cm_id; 2620 ep->com.qp = qp; 2621 ref_qp(ep); 2622 2623 /* bind QP to EP and move to RTS */ 2624 attrs.mpa_attr = ep->mpa_attr; 2625 attrs.max_ird = ep->ird; 2626 attrs.max_ord = ep->ord; 2627 attrs.llp_stream_handle = ep; 2628 attrs.next_state = C4IW_QP_STATE_RTS; 2629 2630 /* bind QP and TID with INIT_WR */ 2631 mask = C4IW_QP_ATTR_NEXT_STATE | 2632 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2633 C4IW_QP_ATTR_MPA_ATTR | 2634 C4IW_QP_ATTR_MAX_IRD | 2635 C4IW_QP_ATTR_MAX_ORD; 2636 2637 err = c4iw_modify_qp(ep->com.qp->rhp, 2638 ep->com.qp, mask, &attrs, 1); 2639 if (err) 2640 goto err1; 2641 err = send_mpa_reply(ep, conn_param->private_data, 2642 conn_param->private_data_len); 2643 if (err) 2644 goto err1; 2645 2646 state_set(&ep->com, FPDU_MODE); 2647 established_upcall(ep); 2648 c4iw_put_ep(&ep->com); 2649 return 0; 2650 err1: 2651 ep->com.cm_id = NULL; 2652 cm_id->rem_ref(cm_id); 2653 err: 2654 c4iw_put_ep(&ep->com); 2655 return err; 2656 } 2657 2658 static int pick_local_ipaddrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2659 { 2660 struct in_device *ind; 2661 int found = 0; 2662 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2663 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2664 2665 ind = in_dev_get(dev->rdev.lldi.ports[0]); 2666 if (!ind) 2667 return -EADDRNOTAVAIL; 2668 for_primary_ifa(ind) { 2669 laddr->sin_addr.s_addr = ifa->ifa_address; 2670 raddr->sin_addr.s_addr = ifa->ifa_address; 2671 found = 1; 2672 break; 2673 } 2674 endfor_ifa(ind); 2675 in_dev_put(ind); 2676 return found ? 0 : -EADDRNOTAVAIL; 2677 } 2678 2679 static int get_lladdr(struct net_device *dev, struct in6_addr *addr, 2680 unsigned char banned_flags) 2681 { 2682 struct inet6_dev *idev; 2683 int err = -EADDRNOTAVAIL; 2684 2685 rcu_read_lock(); 2686 idev = __in6_dev_get(dev); 2687 if (idev != NULL) { 2688 struct inet6_ifaddr *ifp; 2689 2690 read_lock_bh(&idev->lock); 2691 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2692 if (ifp->scope == IFA_LINK && 2693 !(ifp->flags & banned_flags)) { 2694 memcpy(addr, &ifp->addr, 16); 2695 err = 0; 2696 break; 2697 } 2698 } 2699 read_unlock_bh(&idev->lock); 2700 } 2701 rcu_read_unlock(); 2702 return err; 2703 } 2704 2705 static int pick_local_ip6addrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2706 { 2707 struct in6_addr uninitialized_var(addr); 2708 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2709 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&cm_id->remote_addr; 2710 2711 if (get_lladdr(dev->rdev.lldi.ports[0], &addr, IFA_F_TENTATIVE)) { 2712 memcpy(la6->sin6_addr.s6_addr, &addr, 16); 2713 memcpy(ra6->sin6_addr.s6_addr, &addr, 16); 2714 return 0; 2715 } 2716 return -EADDRNOTAVAIL; 2717 } 2718 2719 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2720 { 2721 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2722 struct c4iw_ep *ep; 2723 int err = 0; 2724 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2725 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2726 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2727 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 2728 &cm_id->remote_addr; 2729 __u8 *ra; 2730 int iptype; 2731 2732 if ((conn_param->ord > c4iw_max_read_depth) || 2733 (conn_param->ird > c4iw_max_read_depth)) { 2734 err = -EINVAL; 2735 goto out; 2736 } 2737 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2738 if (!ep) { 2739 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2740 err = -ENOMEM; 2741 goto out; 2742 } 2743 init_timer(&ep->timer); 2744 ep->plen = conn_param->private_data_len; 2745 if (ep->plen) 2746 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2747 conn_param->private_data, ep->plen); 2748 ep->ird = conn_param->ird; 2749 ep->ord = conn_param->ord; 2750 2751 if (peer2peer && ep->ord == 0) 2752 ep->ord = 1; 2753 2754 cm_id->add_ref(cm_id); 2755 ep->com.dev = dev; 2756 ep->com.cm_id = cm_id; 2757 ep->com.qp = get_qhp(dev, conn_param->qpn); 2758 if (!ep->com.qp) { 2759 PDBG("%s qpn 0x%x not found!\n", __func__, conn_param->qpn); 2760 err = -EINVAL; 2761 goto fail2; 2762 } 2763 ref_qp(ep); 2764 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2765 ep->com.qp, cm_id); 2766 2767 /* 2768 * Allocate an active TID to initiate a TCP connection. 2769 */ 2770 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2771 if (ep->atid == -1) { 2772 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2773 err = -ENOMEM; 2774 goto fail2; 2775 } 2776 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2777 2778 if (cm_id->remote_addr.ss_family == AF_INET) { 2779 iptype = 4; 2780 ra = (__u8 *)&raddr->sin_addr; 2781 2782 /* 2783 * Handle loopback requests to INADDR_ANY. 2784 */ 2785 if ((__force int)raddr->sin_addr.s_addr == INADDR_ANY) { 2786 err = pick_local_ipaddrs(dev, cm_id); 2787 if (err) 2788 goto fail2; 2789 } 2790 2791 /* find a route */ 2792 PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n", 2793 __func__, &laddr->sin_addr, ntohs(laddr->sin_port), 2794 ra, ntohs(raddr->sin_port)); 2795 ep->dst = find_route(dev, laddr->sin_addr.s_addr, 2796 raddr->sin_addr.s_addr, laddr->sin_port, 2797 raddr->sin_port, 0); 2798 } else { 2799 iptype = 6; 2800 ra = (__u8 *)&raddr6->sin6_addr; 2801 2802 /* 2803 * Handle loopback requests to INADDR_ANY. 2804 */ 2805 if (ipv6_addr_type(&raddr6->sin6_addr) == IPV6_ADDR_ANY) { 2806 err = pick_local_ip6addrs(dev, cm_id); 2807 if (err) 2808 goto fail2; 2809 } 2810 2811 /* find a route */ 2812 PDBG("%s saddr %pI6 sport 0x%x raddr %pI6 rport 0x%x\n", 2813 __func__, laddr6->sin6_addr.s6_addr, 2814 ntohs(laddr6->sin6_port), 2815 raddr6->sin6_addr.s6_addr, ntohs(raddr6->sin6_port)); 2816 ep->dst = find_route6(dev, laddr6->sin6_addr.s6_addr, 2817 raddr6->sin6_addr.s6_addr, 2818 laddr6->sin6_port, raddr6->sin6_port, 0, 2819 raddr6->sin6_scope_id); 2820 } 2821 if (!ep->dst) { 2822 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2823 err = -EHOSTUNREACH; 2824 goto fail3; 2825 } 2826 2827 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, true); 2828 if (err) { 2829 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2830 goto fail4; 2831 } 2832 2833 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2834 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2835 ep->l2t->idx); 2836 2837 state_set(&ep->com, CONNECTING); 2838 ep->tos = 0; 2839 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2840 sizeof(ep->com.local_addr)); 2841 memcpy(&ep->com.remote_addr, &cm_id->remote_addr, 2842 sizeof(ep->com.remote_addr)); 2843 2844 /* send connect request to rnic */ 2845 err = send_connect(ep); 2846 if (!err) 2847 goto out; 2848 2849 cxgb4_l2t_release(ep->l2t); 2850 fail4: 2851 dst_release(ep->dst); 2852 fail3: 2853 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 2854 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2855 fail2: 2856 cm_id->rem_ref(cm_id); 2857 c4iw_put_ep(&ep->com); 2858 out: 2859 return err; 2860 } 2861 2862 static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2863 { 2864 int err; 2865 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ep->com.local_addr; 2866 2867 c4iw_init_wr_wait(&ep->com.wr_wait); 2868 err = cxgb4_create_server6(ep->com.dev->rdev.lldi.ports[0], 2869 ep->stid, &sin6->sin6_addr, 2870 sin6->sin6_port, 2871 ep->com.dev->rdev.lldi.rxq_ids[0]); 2872 if (!err) 2873 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2874 &ep->com.wr_wait, 2875 0, 0, __func__); 2876 if (err) 2877 pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n", 2878 err, ep->stid, 2879 sin6->sin6_addr.s6_addr, ntohs(sin6->sin6_port)); 2880 return err; 2881 } 2882 2883 static int create_server4(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2884 { 2885 int err; 2886 struct sockaddr_in *sin = (struct sockaddr_in *)&ep->com.local_addr; 2887 2888 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2889 do { 2890 err = cxgb4_create_server_filter( 2891 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2892 sin->sin_addr.s_addr, sin->sin_port, 0, 2893 ep->com.dev->rdev.lldi.rxq_ids[0], 0, 0); 2894 if (err == -EBUSY) { 2895 set_current_state(TASK_UNINTERRUPTIBLE); 2896 schedule_timeout(usecs_to_jiffies(100)); 2897 } 2898 } while (err == -EBUSY); 2899 } else { 2900 c4iw_init_wr_wait(&ep->com.wr_wait); 2901 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], 2902 ep->stid, sin->sin_addr.s_addr, sin->sin_port, 2903 0, ep->com.dev->rdev.lldi.rxq_ids[0]); 2904 if (!err) 2905 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2906 &ep->com.wr_wait, 2907 0, 0, __func__); 2908 } 2909 if (err) 2910 pr_err("cxgb4_create_server/filter failed err %d stid %d laddr %pI4 lport %d\n" 2911 , err, ep->stid, 2912 &sin->sin_addr, ntohs(sin->sin_port)); 2913 return err; 2914 } 2915 2916 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2917 { 2918 int err = 0; 2919 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2920 struct c4iw_listen_ep *ep; 2921 2922 might_sleep(); 2923 2924 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2925 if (!ep) { 2926 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2927 err = -ENOMEM; 2928 goto fail1; 2929 } 2930 PDBG("%s ep %p\n", __func__, ep); 2931 cm_id->add_ref(cm_id); 2932 ep->com.cm_id = cm_id; 2933 ep->com.dev = dev; 2934 ep->backlog = backlog; 2935 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2936 sizeof(ep->com.local_addr)); 2937 2938 /* 2939 * Allocate a server TID. 2940 */ 2941 if (dev->rdev.lldi.enable_fw_ofld_conn) 2942 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, 2943 cm_id->local_addr.ss_family, ep); 2944 else 2945 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, 2946 cm_id->local_addr.ss_family, ep); 2947 2948 if (ep->stid == -1) { 2949 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2950 err = -ENOMEM; 2951 goto fail2; 2952 } 2953 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2954 state_set(&ep->com, LISTEN); 2955 if (ep->com.local_addr.ss_family == AF_INET) 2956 err = create_server4(dev, ep); 2957 else 2958 err = create_server6(dev, ep); 2959 if (!err) { 2960 cm_id->provider_data = ep; 2961 goto out; 2962 } 2963 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 2964 ep->com.local_addr.ss_family); 2965 fail2: 2966 cm_id->rem_ref(cm_id); 2967 c4iw_put_ep(&ep->com); 2968 fail1: 2969 out: 2970 return err; 2971 } 2972 2973 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2974 { 2975 int err; 2976 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2977 2978 PDBG("%s ep %p\n", __func__, ep); 2979 2980 might_sleep(); 2981 state_set(&ep->com, DEAD); 2982 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn && 2983 ep->com.local_addr.ss_family == AF_INET) { 2984 err = cxgb4_remove_server_filter( 2985 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2986 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2987 } else { 2988 c4iw_init_wr_wait(&ep->com.wr_wait); 2989 err = cxgb4_remove_server( 2990 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2991 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2992 if (err) 2993 goto done; 2994 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 2995 0, 0, __func__); 2996 } 2997 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 2998 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 2999 ep->com.local_addr.ss_family); 3000 done: 3001 cm_id->rem_ref(cm_id); 3002 c4iw_put_ep(&ep->com); 3003 return err; 3004 } 3005 3006 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 3007 { 3008 int ret = 0; 3009 int close = 0; 3010 int fatal = 0; 3011 struct c4iw_rdev *rdev; 3012 3013 mutex_lock(&ep->com.mutex); 3014 3015 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 3016 states[ep->com.state], abrupt); 3017 3018 rdev = &ep->com.dev->rdev; 3019 if (c4iw_fatal_error(rdev)) { 3020 fatal = 1; 3021 close_complete_upcall(ep); 3022 ep->com.state = DEAD; 3023 } 3024 switch (ep->com.state) { 3025 case MPA_REQ_WAIT: 3026 case MPA_REQ_SENT: 3027 case MPA_REQ_RCVD: 3028 case MPA_REP_SENT: 3029 case FPDU_MODE: 3030 close = 1; 3031 if (abrupt) 3032 ep->com.state = ABORTING; 3033 else { 3034 ep->com.state = CLOSING; 3035 start_ep_timer(ep); 3036 } 3037 set_bit(CLOSE_SENT, &ep->com.flags); 3038 break; 3039 case CLOSING: 3040 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 3041 close = 1; 3042 if (abrupt) { 3043 stop_ep_timer(ep); 3044 ep->com.state = ABORTING; 3045 } else 3046 ep->com.state = MORIBUND; 3047 } 3048 break; 3049 case MORIBUND: 3050 case ABORTING: 3051 case DEAD: 3052 PDBG("%s ignoring disconnect ep %p state %u\n", 3053 __func__, ep, ep->com.state); 3054 break; 3055 default: 3056 BUG(); 3057 break; 3058 } 3059 3060 if (close) { 3061 if (abrupt) { 3062 set_bit(EP_DISC_ABORT, &ep->com.history); 3063 close_complete_upcall(ep); 3064 ret = send_abort(ep, NULL, gfp); 3065 } else { 3066 set_bit(EP_DISC_CLOSE, &ep->com.history); 3067 ret = send_halfclose(ep, gfp); 3068 } 3069 if (ret) 3070 fatal = 1; 3071 } 3072 mutex_unlock(&ep->com.mutex); 3073 if (fatal) 3074 release_ep_resources(ep); 3075 return ret; 3076 } 3077 3078 static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3079 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3080 { 3081 struct c4iw_ep *ep; 3082 int atid = be32_to_cpu(req->tid); 3083 3084 ep = (struct c4iw_ep *)lookup_atid(dev->rdev.lldi.tids, 3085 (__force u32) req->tid); 3086 if (!ep) 3087 return; 3088 3089 switch (req->retval) { 3090 case FW_ENOMEM: 3091 set_bit(ACT_RETRY_NOMEM, &ep->com.history); 3092 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3093 send_fw_act_open_req(ep, atid); 3094 return; 3095 } 3096 case FW_EADDRINUSE: 3097 set_bit(ACT_RETRY_INUSE, &ep->com.history); 3098 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3099 send_fw_act_open_req(ep, atid); 3100 return; 3101 } 3102 break; 3103 default: 3104 pr_info("%s unexpected ofld conn wr retval %d\n", 3105 __func__, req->retval); 3106 break; 3107 } 3108 pr_err("active ofld_connect_wr failure %d atid %d\n", 3109 req->retval, atid); 3110 mutex_lock(&dev->rdev.stats.lock); 3111 dev->rdev.stats.act_ofld_conn_fails++; 3112 mutex_unlock(&dev->rdev.stats.lock); 3113 connect_reply_upcall(ep, status2errno(req->retval)); 3114 state_set(&ep->com, DEAD); 3115 remove_handle(dev, &dev->atid_idr, atid); 3116 cxgb4_free_atid(dev->rdev.lldi.tids, atid); 3117 dst_release(ep->dst); 3118 cxgb4_l2t_release(ep->l2t); 3119 c4iw_put_ep(&ep->com); 3120 } 3121 3122 static void passive_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3123 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3124 { 3125 struct sk_buff *rpl_skb; 3126 struct cpl_pass_accept_req *cpl; 3127 int ret; 3128 3129 rpl_skb = (struct sk_buff *)(unsigned long)req->cookie; 3130 BUG_ON(!rpl_skb); 3131 if (req->retval) { 3132 PDBG("%s passive open failure %d\n", __func__, req->retval); 3133 mutex_lock(&dev->rdev.stats.lock); 3134 dev->rdev.stats.pas_ofld_conn_fails++; 3135 mutex_unlock(&dev->rdev.stats.lock); 3136 kfree_skb(rpl_skb); 3137 } else { 3138 cpl = (struct cpl_pass_accept_req *)cplhdr(rpl_skb); 3139 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 3140 (__force u32) htonl( 3141 (__force u32) req->tid))); 3142 ret = pass_accept_req(dev, rpl_skb); 3143 if (!ret) 3144 kfree_skb(rpl_skb); 3145 } 3146 return; 3147 } 3148 3149 static int deferred_fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3150 { 3151 struct cpl_fw6_msg *rpl = cplhdr(skb); 3152 struct cpl_fw6_msg_ofld_connection_wr_rpl *req; 3153 3154 switch (rpl->type) { 3155 case FW6_TYPE_CQE: 3156 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 3157 break; 3158 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3159 req = (struct cpl_fw6_msg_ofld_connection_wr_rpl *)rpl->data; 3160 switch (req->t_state) { 3161 case TCP_SYN_SENT: 3162 active_ofld_conn_reply(dev, skb, req); 3163 break; 3164 case TCP_SYN_RECV: 3165 passive_ofld_conn_reply(dev, skb, req); 3166 break; 3167 default: 3168 pr_err("%s unexpected ofld conn wr state %d\n", 3169 __func__, req->t_state); 3170 break; 3171 } 3172 break; 3173 } 3174 return 0; 3175 } 3176 3177 static void build_cpl_pass_accept_req(struct sk_buff *skb, int stid , u8 tos) 3178 { 3179 u32 l2info; 3180 u16 vlantag, len, hdr_len, eth_hdr_len; 3181 u8 intf; 3182 struct cpl_rx_pkt *cpl = cplhdr(skb); 3183 struct cpl_pass_accept_req *req; 3184 struct tcp_options_received tmp_opt; 3185 struct c4iw_dev *dev; 3186 3187 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3188 /* Store values from cpl_rx_pkt in temporary location. */ 3189 vlantag = (__force u16) cpl->vlan; 3190 len = (__force u16) cpl->len; 3191 l2info = (__force u32) cpl->l2info; 3192 hdr_len = (__force u16) cpl->hdr_len; 3193 intf = cpl->iff; 3194 3195 __skb_pull(skb, sizeof(*req) + sizeof(struct rss_header)); 3196 3197 /* 3198 * We need to parse the TCP options from SYN packet. 3199 * to generate cpl_pass_accept_req. 3200 */ 3201 memset(&tmp_opt, 0, sizeof(tmp_opt)); 3202 tcp_clear_options(&tmp_opt); 3203 tcp_parse_options(skb, &tmp_opt, 0, NULL); 3204 3205 req = (struct cpl_pass_accept_req *)__skb_push(skb, sizeof(*req)); 3206 memset(req, 0, sizeof(*req)); 3207 req->l2info = cpu_to_be16(V_SYN_INTF(intf) | 3208 V_SYN_MAC_IDX(G_RX_MACIDX( 3209 (__force int) htonl(l2info))) | 3210 F_SYN_XACT_MATCH); 3211 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3212 G_RX_ETHHDR_LEN((__force int) htonl(l2info)) : 3213 G_RX_T5_ETHHDR_LEN((__force int) htonl(l2info)); 3214 req->hdr_len = cpu_to_be32(V_SYN_RX_CHAN(G_RX_CHAN( 3215 (__force int) htonl(l2info))) | 3216 V_TCP_HDR_LEN(G_RX_TCPHDR_LEN( 3217 (__force int) htons(hdr_len))) | 3218 V_IP_HDR_LEN(G_RX_IPHDR_LEN( 3219 (__force int) htons(hdr_len))) | 3220 V_ETH_HDR_LEN(G_RX_ETHHDR_LEN(eth_hdr_len))); 3221 req->vlan = (__force __be16) vlantag; 3222 req->len = (__force __be16) len; 3223 req->tos_stid = cpu_to_be32(PASS_OPEN_TID(stid) | 3224 PASS_OPEN_TOS(tos)); 3225 req->tcpopt.mss = htons(tmp_opt.mss_clamp); 3226 if (tmp_opt.wscale_ok) 3227 req->tcpopt.wsf = tmp_opt.snd_wscale; 3228 req->tcpopt.tstamp = tmp_opt.saw_tstamp; 3229 if (tmp_opt.sack_ok) 3230 req->tcpopt.sack = 1; 3231 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 0)); 3232 return; 3233 } 3234 3235 static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, 3236 __be32 laddr, __be16 lport, 3237 __be32 raddr, __be16 rport, 3238 u32 rcv_isn, u32 filter, u16 window, 3239 u32 rss_qid, u8 port_id) 3240 { 3241 struct sk_buff *req_skb; 3242 struct fw_ofld_connection_wr *req; 3243 struct cpl_pass_accept_req *cpl = cplhdr(skb); 3244 3245 req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); 3246 req = (struct fw_ofld_connection_wr *)__skb_put(req_skb, sizeof(*req)); 3247 memset(req, 0, sizeof(*req)); 3248 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL(1)); 3249 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 3250 req->le.version_cpl = htonl(F_FW_OFLD_CONNECTION_WR_CPL); 3251 req->le.filter = (__force __be32) filter; 3252 req->le.lport = lport; 3253 req->le.pport = rport; 3254 req->le.u.ipv4.lip = laddr; 3255 req->le.u.ipv4.pip = raddr; 3256 req->tcb.rcv_nxt = htonl(rcv_isn + 1); 3257 req->tcb.rcv_adv = htons(window); 3258 req->tcb.t_state_to_astid = 3259 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_RECV) | 3260 V_FW_OFLD_CONNECTION_WR_RCV_SCALE(cpl->tcpopt.wsf) | 3261 V_FW_OFLD_CONNECTION_WR_ASTID( 3262 GET_PASS_OPEN_TID(ntohl(cpl->tos_stid)))); 3263 3264 /* 3265 * We store the qid in opt2 which will be used by the firmware 3266 * to send us the wr response. 3267 */ 3268 req->tcb.opt2 = htonl(V_RSS_QUEUE(rss_qid)); 3269 3270 /* 3271 * We initialize the MSS index in TCB to 0xF. 3272 * So that when driver sends cpl_pass_accept_rpl 3273 * TCB picks up the correct value. If this was 0 3274 * TP will ignore any value > 0 for MSS index. 3275 */ 3276 req->tcb.opt0 = cpu_to_be64(V_MSS_IDX(0xF)); 3277 req->cookie = (unsigned long)skb; 3278 3279 set_wr_txq(req_skb, CPL_PRIORITY_CONTROL, port_id); 3280 cxgb4_ofld_send(dev->rdev.lldi.ports[0], req_skb); 3281 } 3282 3283 /* 3284 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt 3285 * messages when a filter is being used instead of server to 3286 * redirect a syn packet. When packets hit filter they are redirected 3287 * to the offload queue and driver tries to establish the connection 3288 * using firmware work request. 3289 */ 3290 static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) 3291 { 3292 int stid; 3293 unsigned int filter; 3294 struct ethhdr *eh = NULL; 3295 struct vlan_ethhdr *vlan_eh = NULL; 3296 struct iphdr *iph; 3297 struct tcphdr *tcph; 3298 struct rss_header *rss = (void *)skb->data; 3299 struct cpl_rx_pkt *cpl = (void *)skb->data; 3300 struct cpl_pass_accept_req *req = (void *)(rss + 1); 3301 struct l2t_entry *e; 3302 struct dst_entry *dst; 3303 struct c4iw_ep *lep; 3304 u16 window; 3305 struct port_info *pi; 3306 struct net_device *pdev; 3307 u16 rss_qid, eth_hdr_len; 3308 int step; 3309 u32 tx_chan; 3310 struct neighbour *neigh; 3311 3312 /* Drop all non-SYN packets */ 3313 if (!(cpl->l2info & cpu_to_be32(F_RXF_SYN))) 3314 goto reject; 3315 3316 /* 3317 * Drop all packets which did not hit the filter. 3318 * Unlikely to happen. 3319 */ 3320 if (!(rss->filter_hit && rss->filter_tid)) 3321 goto reject; 3322 3323 /* 3324 * Calculate the server tid from filter hit index from cpl_rx_pkt. 3325 */ 3326 stid = (__force int) cpu_to_be32((__force u32) rss->hash_val) 3327 - dev->rdev.lldi.tids->sftid_base 3328 + dev->rdev.lldi.tids->nstids; 3329 3330 lep = (struct c4iw_ep *)lookup_stid(dev->rdev.lldi.tids, stid); 3331 if (!lep) { 3332 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 3333 goto reject; 3334 } 3335 3336 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3337 G_RX_ETHHDR_LEN(htonl(cpl->l2info)) : 3338 G_RX_T5_ETHHDR_LEN(htonl(cpl->l2info)); 3339 if (eth_hdr_len == ETH_HLEN) { 3340 eh = (struct ethhdr *)(req + 1); 3341 iph = (struct iphdr *)(eh + 1); 3342 } else { 3343 vlan_eh = (struct vlan_ethhdr *)(req + 1); 3344 iph = (struct iphdr *)(vlan_eh + 1); 3345 skb->vlan_tci = ntohs(cpl->vlan); 3346 } 3347 3348 if (iph->version != 0x4) 3349 goto reject; 3350 3351 tcph = (struct tcphdr *)(iph + 1); 3352 skb_set_network_header(skb, (void *)iph - (void *)rss); 3353 skb_set_transport_header(skb, (void *)tcph - (void *)rss); 3354 skb_get(skb); 3355 3356 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__, 3357 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3358 ntohs(tcph->source), iph->tos); 3359 3360 dst = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3361 iph->tos); 3362 if (!dst) { 3363 pr_err("%s - failed to find dst entry!\n", 3364 __func__); 3365 goto reject; 3366 } 3367 neigh = dst_neigh_lookup_skb(dst, skb); 3368 3369 if (!neigh) { 3370 pr_err("%s - failed to allocate neigh!\n", 3371 __func__); 3372 goto free_dst; 3373 } 3374 3375 if (neigh->dev->flags & IFF_LOOPBACK) { 3376 pdev = ip_dev_find(&init_net, iph->daddr); 3377 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3378 pdev, 0); 3379 pi = (struct port_info *)netdev_priv(pdev); 3380 tx_chan = cxgb4_port_chan(pdev); 3381 dev_put(pdev); 3382 } else { 3383 pdev = get_real_dev(neigh->dev); 3384 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3385 pdev, 0); 3386 pi = (struct port_info *)netdev_priv(pdev); 3387 tx_chan = cxgb4_port_chan(pdev); 3388 } 3389 if (!e) { 3390 pr_err("%s - failed to allocate l2t entry!\n", 3391 __func__); 3392 goto free_dst; 3393 } 3394 3395 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; 3396 rss_qid = dev->rdev.lldi.rxq_ids[pi->port_id * step]; 3397 window = (__force u16) htons((__force u16)tcph->window); 3398 3399 /* Calcuate filter portion for LE region. */ 3400 filter = (__force unsigned int) cpu_to_be32(select_ntuple(dev, dst, e)); 3401 3402 /* 3403 * Synthesize the cpl_pass_accept_req. We have everything except the 3404 * TID. Once firmware sends a reply with TID we update the TID field 3405 * in cpl and pass it through the regular cpl_pass_accept_req path. 3406 */ 3407 build_cpl_pass_accept_req(skb, stid, iph->tos); 3408 send_fw_pass_open_req(dev, skb, iph->daddr, tcph->dest, iph->saddr, 3409 tcph->source, ntohl(tcph->seq), filter, window, 3410 rss_qid, pi->port_id); 3411 cxgb4_l2t_release(e); 3412 free_dst: 3413 dst_release(dst); 3414 reject: 3415 return 0; 3416 } 3417 3418 /* 3419 * These are the real handlers that are called from a 3420 * work queue. 3421 */ 3422 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 3423 [CPL_ACT_ESTABLISH] = act_establish, 3424 [CPL_ACT_OPEN_RPL] = act_open_rpl, 3425 [CPL_RX_DATA] = rx_data, 3426 [CPL_ABORT_RPL_RSS] = abort_rpl, 3427 [CPL_ABORT_RPL] = abort_rpl, 3428 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 3429 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 3430 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 3431 [CPL_PASS_ESTABLISH] = pass_establish, 3432 [CPL_PEER_CLOSE] = peer_close, 3433 [CPL_ABORT_REQ_RSS] = peer_abort, 3434 [CPL_CLOSE_CON_RPL] = close_con_rpl, 3435 [CPL_RDMA_TERMINATE] = terminate, 3436 [CPL_FW4_ACK] = fw4_ack, 3437 [CPL_FW6_MSG] = deferred_fw6_msg, 3438 [CPL_RX_PKT] = rx_pkt 3439 }; 3440 3441 static void process_timeout(struct c4iw_ep *ep) 3442 { 3443 struct c4iw_qp_attributes attrs; 3444 int abort = 1; 3445 3446 mutex_lock(&ep->com.mutex); 3447 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 3448 ep->com.state); 3449 set_bit(TIMEDOUT, &ep->com.history); 3450 switch (ep->com.state) { 3451 case MPA_REQ_SENT: 3452 __state_set(&ep->com, ABORTING); 3453 connect_reply_upcall(ep, -ETIMEDOUT); 3454 break; 3455 case MPA_REQ_WAIT: 3456 __state_set(&ep->com, ABORTING); 3457 break; 3458 case CLOSING: 3459 case MORIBUND: 3460 if (ep->com.cm_id && ep->com.qp) { 3461 attrs.next_state = C4IW_QP_STATE_ERROR; 3462 c4iw_modify_qp(ep->com.qp->rhp, 3463 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 3464 &attrs, 1); 3465 } 3466 __state_set(&ep->com, ABORTING); 3467 break; 3468 default: 3469 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3470 __func__, ep, ep->hwtid, ep->com.state); 3471 abort = 0; 3472 } 3473 mutex_unlock(&ep->com.mutex); 3474 if (abort) 3475 abort_connection(ep, NULL, GFP_KERNEL); 3476 c4iw_put_ep(&ep->com); 3477 } 3478 3479 static void process_timedout_eps(void) 3480 { 3481 struct c4iw_ep *ep; 3482 3483 spin_lock_irq(&timeout_lock); 3484 while (!list_empty(&timeout_list)) { 3485 struct list_head *tmp; 3486 3487 tmp = timeout_list.next; 3488 list_del(tmp); 3489 spin_unlock_irq(&timeout_lock); 3490 ep = list_entry(tmp, struct c4iw_ep, entry); 3491 process_timeout(ep); 3492 spin_lock_irq(&timeout_lock); 3493 } 3494 spin_unlock_irq(&timeout_lock); 3495 } 3496 3497 static void process_work(struct work_struct *work) 3498 { 3499 struct sk_buff *skb = NULL; 3500 struct c4iw_dev *dev; 3501 struct cpl_act_establish *rpl; 3502 unsigned int opcode; 3503 int ret; 3504 3505 while ((skb = skb_dequeue(&rxq))) { 3506 rpl = cplhdr(skb); 3507 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3508 opcode = rpl->ot.opcode; 3509 3510 BUG_ON(!work_handlers[opcode]); 3511 ret = work_handlers[opcode](dev, skb); 3512 if (!ret) 3513 kfree_skb(skb); 3514 } 3515 process_timedout_eps(); 3516 } 3517 3518 static DECLARE_WORK(skb_work, process_work); 3519 3520 static void ep_timeout(unsigned long arg) 3521 { 3522 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 3523 int kickit = 0; 3524 3525 spin_lock(&timeout_lock); 3526 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 3527 list_add_tail(&ep->entry, &timeout_list); 3528 kickit = 1; 3529 } 3530 spin_unlock(&timeout_lock); 3531 if (kickit) 3532 queue_work(workq, &skb_work); 3533 } 3534 3535 /* 3536 * All the CM events are handled on a work queue to have a safe context. 3537 */ 3538 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 3539 { 3540 3541 /* 3542 * Save dev in the skb->cb area. 3543 */ 3544 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 3545 3546 /* 3547 * Queue the skb and schedule the worker thread. 3548 */ 3549 skb_queue_tail(&rxq, skb); 3550 queue_work(workq, &skb_work); 3551 return 0; 3552 } 3553 3554 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 3555 { 3556 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 3557 3558 if (rpl->status != CPL_ERR_NONE) { 3559 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 3560 "for tid %u\n", rpl->status, GET_TID(rpl)); 3561 } 3562 kfree_skb(skb); 3563 return 0; 3564 } 3565 3566 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3567 { 3568 struct cpl_fw6_msg *rpl = cplhdr(skb); 3569 struct c4iw_wr_wait *wr_waitp; 3570 int ret; 3571 3572 PDBG("%s type %u\n", __func__, rpl->type); 3573 3574 switch (rpl->type) { 3575 case FW6_TYPE_WR_RPL: 3576 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 3577 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 3578 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 3579 if (wr_waitp) 3580 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 3581 kfree_skb(skb); 3582 break; 3583 case FW6_TYPE_CQE: 3584 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3585 sched(dev, skb); 3586 break; 3587 default: 3588 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 3589 rpl->type); 3590 kfree_skb(skb); 3591 break; 3592 } 3593 return 0; 3594 } 3595 3596 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 3597 { 3598 struct cpl_abort_req_rss *req = cplhdr(skb); 3599 struct c4iw_ep *ep; 3600 struct tid_info *t = dev->rdev.lldi.tids; 3601 unsigned int tid = GET_TID(req); 3602 3603 ep = lookup_tid(t, tid); 3604 if (!ep) { 3605 printk(KERN_WARNING MOD 3606 "Abort on non-existent endpoint, tid %d\n", tid); 3607 kfree_skb(skb); 3608 return 0; 3609 } 3610 if (is_neg_adv_abort(req->status)) { 3611 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 3612 ep->hwtid); 3613 kfree_skb(skb); 3614 return 0; 3615 } 3616 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 3617 ep->com.state); 3618 3619 /* 3620 * Wake up any threads in rdma_init() or rdma_fini(). 3621 * However, if we are on MPAv2 and want to retry with MPAv1 3622 * then, don't wake up yet. 3623 */ 3624 if (mpa_rev == 2 && !ep->tried_with_mpa_v1) { 3625 if (ep->com.state != MPA_REQ_SENT) 3626 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3627 } else 3628 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3629 sched(dev, skb); 3630 return 0; 3631 } 3632 3633 /* 3634 * Most upcalls from the T4 Core go to sched() to 3635 * schedule the processing on a work queue. 3636 */ 3637 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 3638 [CPL_ACT_ESTABLISH] = sched, 3639 [CPL_ACT_OPEN_RPL] = sched, 3640 [CPL_RX_DATA] = sched, 3641 [CPL_ABORT_RPL_RSS] = sched, 3642 [CPL_ABORT_RPL] = sched, 3643 [CPL_PASS_OPEN_RPL] = sched, 3644 [CPL_CLOSE_LISTSRV_RPL] = sched, 3645 [CPL_PASS_ACCEPT_REQ] = sched, 3646 [CPL_PASS_ESTABLISH] = sched, 3647 [CPL_PEER_CLOSE] = sched, 3648 [CPL_CLOSE_CON_RPL] = sched, 3649 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 3650 [CPL_RDMA_TERMINATE] = sched, 3651 [CPL_FW4_ACK] = sched, 3652 [CPL_SET_TCB_RPL] = set_tcb_rpl, 3653 [CPL_FW6_MSG] = fw6_msg, 3654 [CPL_RX_PKT] = sched 3655 }; 3656 3657 int __init c4iw_cm_init(void) 3658 { 3659 spin_lock_init(&timeout_lock); 3660 skb_queue_head_init(&rxq); 3661 3662 workq = create_singlethread_workqueue("iw_cxgb4"); 3663 if (!workq) 3664 return -ENOMEM; 3665 3666 return 0; 3667 } 3668 3669 void __exit c4iw_cm_term(void) 3670 { 3671 WARN_ON(!list_empty(&timeout_list)); 3672 flush_workqueue(workq); 3673 destroy_workqueue(workq); 3674 } 3675