1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 #include <linux/if_vlan.h> 42 43 #include <net/neighbour.h> 44 #include <net/netevent.h> 45 #include <net/route.h> 46 #include <net/tcp.h> 47 #include <net/ip6_route.h> 48 #include <net/addrconf.h> 49 50 #include "iw_cxgb4.h" 51 52 static char *states[] = { 53 "idle", 54 "listen", 55 "connecting", 56 "mpa_wait_req", 57 "mpa_req_sent", 58 "mpa_req_rcvd", 59 "mpa_rep_sent", 60 "fpdu_mode", 61 "aborting", 62 "closing", 63 "moribund", 64 "dead", 65 NULL, 66 }; 67 68 static int nocong; 69 module_param(nocong, int, 0644); 70 MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)"); 71 72 static int enable_ecn; 73 module_param(enable_ecn, int, 0644); 74 MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); 75 76 static int dack_mode = 1; 77 module_param(dack_mode, int, 0644); 78 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 79 80 int c4iw_max_read_depth = 8; 81 module_param(c4iw_max_read_depth, int, 0644); 82 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 83 84 static int enable_tcp_timestamps; 85 module_param(enable_tcp_timestamps, int, 0644); 86 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 87 88 static int enable_tcp_sack; 89 module_param(enable_tcp_sack, int, 0644); 90 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 91 92 static int enable_tcp_window_scaling = 1; 93 module_param(enable_tcp_window_scaling, int, 0644); 94 MODULE_PARM_DESC(enable_tcp_window_scaling, 95 "Enable tcp window scaling (default=1)"); 96 97 int c4iw_debug; 98 module_param(c4iw_debug, int, 0644); 99 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 100 101 static int peer2peer = 1; 102 module_param(peer2peer, int, 0644); 103 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=1)"); 104 105 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 106 module_param(p2p_type, int, 0644); 107 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 108 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 109 110 static int ep_timeout_secs = 60; 111 module_param(ep_timeout_secs, int, 0644); 112 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 113 "in seconds (default=60)"); 114 115 static int mpa_rev = 1; 116 module_param(mpa_rev, int, 0644); 117 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 118 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 119 " compliant (default=1)"); 120 121 static int markers_enabled; 122 module_param(markers_enabled, int, 0644); 123 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 124 125 static int crc_enabled = 1; 126 module_param(crc_enabled, int, 0644); 127 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 128 129 static int rcv_win = 256 * 1024; 130 module_param(rcv_win, int, 0644); 131 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 132 133 static int snd_win = 128 * 1024; 134 module_param(snd_win, int, 0644); 135 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 136 137 static struct workqueue_struct *workq; 138 139 static struct sk_buff_head rxq; 140 141 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 142 static void ep_timeout(unsigned long arg); 143 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 144 145 static LIST_HEAD(timeout_list); 146 static spinlock_t timeout_lock; 147 148 static void deref_qp(struct c4iw_ep *ep) 149 { 150 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 151 clear_bit(QP_REFERENCED, &ep->com.flags); 152 } 153 154 static void ref_qp(struct c4iw_ep *ep) 155 { 156 set_bit(QP_REFERENCED, &ep->com.flags); 157 c4iw_qp_add_ref(&ep->com.qp->ibqp); 158 } 159 160 static void start_ep_timer(struct c4iw_ep *ep) 161 { 162 PDBG("%s ep %p\n", __func__, ep); 163 if (timer_pending(&ep->timer)) { 164 pr_err("%s timer already started! ep %p\n", 165 __func__, ep); 166 return; 167 } 168 clear_bit(TIMEOUT, &ep->com.flags); 169 c4iw_get_ep(&ep->com); 170 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 171 ep->timer.data = (unsigned long)ep; 172 ep->timer.function = ep_timeout; 173 add_timer(&ep->timer); 174 } 175 176 static int stop_ep_timer(struct c4iw_ep *ep) 177 { 178 PDBG("%s ep %p stopping\n", __func__, ep); 179 del_timer_sync(&ep->timer); 180 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 181 c4iw_put_ep(&ep->com); 182 return 0; 183 } 184 return 1; 185 } 186 187 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 188 struct l2t_entry *l2e) 189 { 190 int error = 0; 191 192 if (c4iw_fatal_error(rdev)) { 193 kfree_skb(skb); 194 PDBG("%s - device in error state - dropping\n", __func__); 195 return -EIO; 196 } 197 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 198 if (error < 0) 199 kfree_skb(skb); 200 return error < 0 ? error : 0; 201 } 202 203 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 204 { 205 int error = 0; 206 207 if (c4iw_fatal_error(rdev)) { 208 kfree_skb(skb); 209 PDBG("%s - device in error state - dropping\n", __func__); 210 return -EIO; 211 } 212 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 213 if (error < 0) 214 kfree_skb(skb); 215 return error < 0 ? error : 0; 216 } 217 218 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 219 { 220 struct cpl_tid_release *req; 221 222 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 223 if (!skb) 224 return; 225 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 226 INIT_TP_WR(req, hwtid); 227 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 228 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 229 c4iw_ofld_send(rdev, skb); 230 return; 231 } 232 233 static void set_emss(struct c4iw_ep *ep, u16 opt) 234 { 235 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 236 ep->mss = ep->emss; 237 if (GET_TCPOPT_TSTAMP(opt)) 238 ep->emss -= 12; 239 if (ep->emss < 128) 240 ep->emss = 128; 241 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 242 ep->mss, ep->emss); 243 } 244 245 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 246 { 247 enum c4iw_ep_state state; 248 249 mutex_lock(&epc->mutex); 250 state = epc->state; 251 mutex_unlock(&epc->mutex); 252 return state; 253 } 254 255 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 256 { 257 epc->state = new; 258 } 259 260 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 261 { 262 mutex_lock(&epc->mutex); 263 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 264 __state_set(epc, new); 265 mutex_unlock(&epc->mutex); 266 return; 267 } 268 269 static void *alloc_ep(int size, gfp_t gfp) 270 { 271 struct c4iw_ep_common *epc; 272 273 epc = kzalloc(size, gfp); 274 if (epc) { 275 kref_init(&epc->kref); 276 mutex_init(&epc->mutex); 277 c4iw_init_wr_wait(&epc->wr_wait); 278 } 279 PDBG("%s alloc ep %p\n", __func__, epc); 280 return epc; 281 } 282 283 void _c4iw_free_ep(struct kref *kref) 284 { 285 struct c4iw_ep *ep; 286 287 ep = container_of(kref, struct c4iw_ep, com.kref); 288 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 289 if (test_bit(QP_REFERENCED, &ep->com.flags)) 290 deref_qp(ep); 291 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 292 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 293 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 294 dst_release(ep->dst); 295 cxgb4_l2t_release(ep->l2t); 296 } 297 kfree(ep); 298 } 299 300 static void release_ep_resources(struct c4iw_ep *ep) 301 { 302 set_bit(RELEASE_RESOURCES, &ep->com.flags); 303 c4iw_put_ep(&ep->com); 304 } 305 306 static int status2errno(int status) 307 { 308 switch (status) { 309 case CPL_ERR_NONE: 310 return 0; 311 case CPL_ERR_CONN_RESET: 312 return -ECONNRESET; 313 case CPL_ERR_ARP_MISS: 314 return -EHOSTUNREACH; 315 case CPL_ERR_CONN_TIMEDOUT: 316 return -ETIMEDOUT; 317 case CPL_ERR_TCAM_FULL: 318 return -ENOMEM; 319 case CPL_ERR_CONN_EXIST: 320 return -EADDRINUSE; 321 default: 322 return -EIO; 323 } 324 } 325 326 /* 327 * Try and reuse skbs already allocated... 328 */ 329 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 330 { 331 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 332 skb_trim(skb, 0); 333 skb_get(skb); 334 skb_reset_transport_header(skb); 335 } else { 336 skb = alloc_skb(len, gfp); 337 } 338 t4_set_arp_err_handler(skb, NULL, NULL); 339 return skb; 340 } 341 342 static struct net_device *get_real_dev(struct net_device *egress_dev) 343 { 344 struct net_device *phys_dev = egress_dev; 345 if (egress_dev->priv_flags & IFF_802_1Q_VLAN) 346 phys_dev = vlan_dev_real_dev(egress_dev); 347 return phys_dev; 348 } 349 350 static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev) 351 { 352 int i; 353 354 egress_dev = get_real_dev(egress_dev); 355 for (i = 0; i < dev->rdev.lldi.nports; i++) 356 if (dev->rdev.lldi.ports[i] == egress_dev) 357 return 1; 358 return 0; 359 } 360 361 static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip, 362 __u8 *peer_ip, __be16 local_port, 363 __be16 peer_port, u8 tos, 364 __u32 sin6_scope_id) 365 { 366 struct dst_entry *dst = NULL; 367 368 if (IS_ENABLED(CONFIG_IPV6)) { 369 struct flowi6 fl6; 370 371 memset(&fl6, 0, sizeof(fl6)); 372 memcpy(&fl6.daddr, peer_ip, 16); 373 memcpy(&fl6.saddr, local_ip, 16); 374 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 375 fl6.flowi6_oif = sin6_scope_id; 376 dst = ip6_route_output(&init_net, NULL, &fl6); 377 if (!dst) 378 goto out; 379 if (!our_interface(dev, ip6_dst_idev(dst)->dev) && 380 !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) { 381 dst_release(dst); 382 dst = NULL; 383 } 384 } 385 386 out: 387 return dst; 388 } 389 390 static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip, 391 __be32 peer_ip, __be16 local_port, 392 __be16 peer_port, u8 tos) 393 { 394 struct rtable *rt; 395 struct flowi4 fl4; 396 struct neighbour *n; 397 398 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 399 peer_port, local_port, IPPROTO_TCP, 400 tos, 0); 401 if (IS_ERR(rt)) 402 return NULL; 403 n = dst_neigh_lookup(&rt->dst, &peer_ip); 404 if (!n) 405 return NULL; 406 if (!our_interface(dev, n->dev) && 407 !(n->dev->flags & IFF_LOOPBACK)) { 408 dst_release(&rt->dst); 409 return NULL; 410 } 411 neigh_release(n); 412 return &rt->dst; 413 } 414 415 static void arp_failure_discard(void *handle, struct sk_buff *skb) 416 { 417 PDBG("%s c4iw_dev %p\n", __func__, handle); 418 kfree_skb(skb); 419 } 420 421 /* 422 * Handle an ARP failure for an active open. 423 */ 424 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 425 { 426 printk(KERN_ERR MOD "ARP failure duing connect\n"); 427 kfree_skb(skb); 428 } 429 430 /* 431 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 432 * and send it along. 433 */ 434 static void abort_arp_failure(void *handle, struct sk_buff *skb) 435 { 436 struct c4iw_rdev *rdev = handle; 437 struct cpl_abort_req *req = cplhdr(skb); 438 439 PDBG("%s rdev %p\n", __func__, rdev); 440 req->cmd = CPL_ABORT_NO_RST; 441 c4iw_ofld_send(rdev, skb); 442 } 443 444 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 445 { 446 unsigned int flowclen = 80; 447 struct fw_flowc_wr *flowc; 448 int i; 449 450 skb = get_skb(skb, flowclen, GFP_KERNEL); 451 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 452 453 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 454 FW_FLOWC_WR_NPARAMS(8)); 455 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 456 16)) | FW_WR_FLOWID(ep->hwtid)); 457 458 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 459 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 460 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 461 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 462 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 463 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 464 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 465 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 466 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 467 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 468 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 469 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 470 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 471 flowc->mnemval[6].val = cpu_to_be32(snd_win); 472 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 473 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 474 /* Pad WR to 16 byte boundary */ 475 flowc->mnemval[8].mnemonic = 0; 476 flowc->mnemval[8].val = 0; 477 for (i = 0; i < 9; i++) { 478 flowc->mnemval[i].r4[0] = 0; 479 flowc->mnemval[i].r4[1] = 0; 480 flowc->mnemval[i].r4[2] = 0; 481 } 482 483 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 484 c4iw_ofld_send(&ep->com.dev->rdev, skb); 485 } 486 487 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 488 { 489 struct cpl_close_con_req *req; 490 struct sk_buff *skb; 491 int wrlen = roundup(sizeof *req, 16); 492 493 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 494 skb = get_skb(NULL, wrlen, gfp); 495 if (!skb) { 496 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 497 return -ENOMEM; 498 } 499 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 500 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 501 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 502 memset(req, 0, wrlen); 503 INIT_TP_WR(req, ep->hwtid); 504 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 505 ep->hwtid)); 506 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 507 } 508 509 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 510 { 511 struct cpl_abort_req *req; 512 int wrlen = roundup(sizeof *req, 16); 513 514 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 515 skb = get_skb(skb, wrlen, gfp); 516 if (!skb) { 517 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 518 __func__); 519 return -ENOMEM; 520 } 521 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 522 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 523 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 524 memset(req, 0, wrlen); 525 INIT_TP_WR(req, ep->hwtid); 526 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 527 req->cmd = CPL_ABORT_SEND_RST; 528 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 529 } 530 531 static int send_connect(struct c4iw_ep *ep) 532 { 533 struct cpl_act_open_req *req; 534 struct cpl_t5_act_open_req *t5_req; 535 struct cpl_act_open_req6 *req6; 536 struct cpl_t5_act_open_req6 *t5_req6; 537 struct sk_buff *skb; 538 u64 opt0; 539 u32 opt2; 540 unsigned int mtu_idx; 541 int wscale; 542 int wrlen; 543 int sizev4 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 544 sizeof(struct cpl_act_open_req) : 545 sizeof(struct cpl_t5_act_open_req); 546 int sizev6 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 547 sizeof(struct cpl_act_open_req6) : 548 sizeof(struct cpl_t5_act_open_req6); 549 struct sockaddr_in *la = (struct sockaddr_in *)&ep->com.local_addr; 550 struct sockaddr_in *ra = (struct sockaddr_in *)&ep->com.remote_addr; 551 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 552 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 553 554 wrlen = (ep->com.remote_addr.ss_family == AF_INET) ? 555 roundup(sizev4, 16) : 556 roundup(sizev6, 16); 557 558 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 559 560 skb = get_skb(NULL, wrlen, GFP_KERNEL); 561 if (!skb) { 562 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 563 __func__); 564 return -ENOMEM; 565 } 566 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 567 568 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 569 wscale = compute_wscale(rcv_win); 570 opt0 = (nocong ? NO_CONG(1) : 0) | 571 KEEP_ALIVE(1) | 572 DELACK(1) | 573 WND_SCALE(wscale) | 574 MSS_IDX(mtu_idx) | 575 L2T_IDX(ep->l2t->idx) | 576 TX_CHAN(ep->tx_chan) | 577 SMAC_SEL(ep->smac_idx) | 578 DSCP(ep->tos) | 579 ULP_MODE(ULP_MODE_TCPDDP) | 580 RCV_BUFSIZ(rcv_win>>10); 581 opt2 = RX_CHANNEL(0) | 582 CCTRL_ECN(enable_ecn) | 583 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 584 if (enable_tcp_timestamps) 585 opt2 |= TSTAMPS_EN(1); 586 if (enable_tcp_sack) 587 opt2 |= SACK_EN(1); 588 if (wscale && enable_tcp_window_scaling) 589 opt2 |= WND_SCALE_EN(1); 590 if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { 591 opt2 |= T5_OPT_2_VALID; 592 opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE); 593 } 594 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 595 596 if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { 597 if (ep->com.remote_addr.ss_family == AF_INET) { 598 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 599 INIT_TP_WR(req, 0); 600 OPCODE_TID(req) = cpu_to_be32( 601 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 602 ((ep->rss_qid << 14) | ep->atid))); 603 req->local_port = la->sin_port; 604 req->peer_port = ra->sin_port; 605 req->local_ip = la->sin_addr.s_addr; 606 req->peer_ip = ra->sin_addr.s_addr; 607 req->opt0 = cpu_to_be64(opt0); 608 req->params = cpu_to_be32(cxgb4_select_ntuple( 609 ep->com.dev->rdev.lldi.ports[0], 610 ep->l2t)); 611 req->opt2 = cpu_to_be32(opt2); 612 } else { 613 req6 = (struct cpl_act_open_req6 *)skb_put(skb, wrlen); 614 615 INIT_TP_WR(req6, 0); 616 OPCODE_TID(req6) = cpu_to_be32( 617 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 618 ((ep->rss_qid<<14)|ep->atid))); 619 req6->local_port = la6->sin6_port; 620 req6->peer_port = ra6->sin6_port; 621 req6->local_ip_hi = *((__be64 *) 622 (la6->sin6_addr.s6_addr)); 623 req6->local_ip_lo = *((__be64 *) 624 (la6->sin6_addr.s6_addr + 8)); 625 req6->peer_ip_hi = *((__be64 *) 626 (ra6->sin6_addr.s6_addr)); 627 req6->peer_ip_lo = *((__be64 *) 628 (ra6->sin6_addr.s6_addr + 8)); 629 req6->opt0 = cpu_to_be64(opt0); 630 req6->params = cpu_to_be32(cxgb4_select_ntuple( 631 ep->com.dev->rdev.lldi.ports[0], 632 ep->l2t)); 633 req6->opt2 = cpu_to_be32(opt2); 634 } 635 } else { 636 if (ep->com.remote_addr.ss_family == AF_INET) { 637 t5_req = (struct cpl_t5_act_open_req *) 638 skb_put(skb, wrlen); 639 INIT_TP_WR(t5_req, 0); 640 OPCODE_TID(t5_req) = cpu_to_be32( 641 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 642 ((ep->rss_qid << 14) | ep->atid))); 643 t5_req->local_port = la->sin_port; 644 t5_req->peer_port = ra->sin_port; 645 t5_req->local_ip = la->sin_addr.s_addr; 646 t5_req->peer_ip = ra->sin_addr.s_addr; 647 t5_req->opt0 = cpu_to_be64(opt0); 648 t5_req->params = cpu_to_be64(V_FILTER_TUPLE( 649 cxgb4_select_ntuple( 650 ep->com.dev->rdev.lldi.ports[0], 651 ep->l2t))); 652 t5_req->opt2 = cpu_to_be32(opt2); 653 } else { 654 t5_req6 = (struct cpl_t5_act_open_req6 *) 655 skb_put(skb, wrlen); 656 INIT_TP_WR(t5_req6, 0); 657 OPCODE_TID(t5_req6) = cpu_to_be32( 658 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 659 ((ep->rss_qid<<14)|ep->atid))); 660 t5_req6->local_port = la6->sin6_port; 661 t5_req6->peer_port = ra6->sin6_port; 662 t5_req6->local_ip_hi = *((__be64 *) 663 (la6->sin6_addr.s6_addr)); 664 t5_req6->local_ip_lo = *((__be64 *) 665 (la6->sin6_addr.s6_addr + 8)); 666 t5_req6->peer_ip_hi = *((__be64 *) 667 (ra6->sin6_addr.s6_addr)); 668 t5_req6->peer_ip_lo = *((__be64 *) 669 (ra6->sin6_addr.s6_addr + 8)); 670 t5_req6->opt0 = cpu_to_be64(opt0); 671 t5_req6->params = (__force __be64)cpu_to_be32( 672 cxgb4_select_ntuple( 673 ep->com.dev->rdev.lldi.ports[0], 674 ep->l2t)); 675 t5_req6->opt2 = cpu_to_be32(opt2); 676 } 677 } 678 679 set_bit(ACT_OPEN_REQ, &ep->com.history); 680 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 681 } 682 683 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 684 u8 mpa_rev_to_use) 685 { 686 int mpalen, wrlen; 687 struct fw_ofld_tx_data_wr *req; 688 struct mpa_message *mpa; 689 struct mpa_v2_conn_params mpa_v2_params; 690 691 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 692 693 BUG_ON(skb_cloned(skb)); 694 695 mpalen = sizeof(*mpa) + ep->plen; 696 if (mpa_rev_to_use == 2) 697 mpalen += sizeof(struct mpa_v2_conn_params); 698 wrlen = roundup(mpalen + sizeof *req, 16); 699 skb = get_skb(skb, wrlen, GFP_KERNEL); 700 if (!skb) { 701 connect_reply_upcall(ep, -ENOMEM); 702 return; 703 } 704 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 705 706 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 707 memset(req, 0, wrlen); 708 req->op_to_immdlen = cpu_to_be32( 709 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 710 FW_WR_COMPL(1) | 711 FW_WR_IMMDLEN(mpalen)); 712 req->flowid_len16 = cpu_to_be32( 713 FW_WR_FLOWID(ep->hwtid) | 714 FW_WR_LEN16(wrlen >> 4)); 715 req->plen = cpu_to_be32(mpalen); 716 req->tunnel_to_proxy = cpu_to_be32( 717 FW_OFLD_TX_DATA_WR_FLUSH(1) | 718 FW_OFLD_TX_DATA_WR_SHOVE(1)); 719 720 mpa = (struct mpa_message *)(req + 1); 721 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 722 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 723 (markers_enabled ? MPA_MARKERS : 0) | 724 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 725 mpa->private_data_size = htons(ep->plen); 726 mpa->revision = mpa_rev_to_use; 727 if (mpa_rev_to_use == 1) { 728 ep->tried_with_mpa_v1 = 1; 729 ep->retry_with_mpa_v1 = 0; 730 } 731 732 if (mpa_rev_to_use == 2) { 733 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 734 sizeof (struct mpa_v2_conn_params)); 735 mpa_v2_params.ird = htons((u16)ep->ird); 736 mpa_v2_params.ord = htons((u16)ep->ord); 737 738 if (peer2peer) { 739 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 740 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 741 mpa_v2_params.ord |= 742 htons(MPA_V2_RDMA_WRITE_RTR); 743 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 744 mpa_v2_params.ord |= 745 htons(MPA_V2_RDMA_READ_RTR); 746 } 747 memcpy(mpa->private_data, &mpa_v2_params, 748 sizeof(struct mpa_v2_conn_params)); 749 750 if (ep->plen) 751 memcpy(mpa->private_data + 752 sizeof(struct mpa_v2_conn_params), 753 ep->mpa_pkt + sizeof(*mpa), ep->plen); 754 } else 755 if (ep->plen) 756 memcpy(mpa->private_data, 757 ep->mpa_pkt + sizeof(*mpa), ep->plen); 758 759 /* 760 * Reference the mpa skb. This ensures the data area 761 * will remain in memory until the hw acks the tx. 762 * Function fw4_ack() will deref it. 763 */ 764 skb_get(skb); 765 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 766 BUG_ON(ep->mpa_skb); 767 ep->mpa_skb = skb; 768 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 769 start_ep_timer(ep); 770 __state_set(&ep->com, MPA_REQ_SENT); 771 ep->mpa_attr.initiator = 1; 772 ep->snd_seq += mpalen; 773 return; 774 } 775 776 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 777 { 778 int mpalen, wrlen; 779 struct fw_ofld_tx_data_wr *req; 780 struct mpa_message *mpa; 781 struct sk_buff *skb; 782 struct mpa_v2_conn_params mpa_v2_params; 783 784 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 785 786 mpalen = sizeof(*mpa) + plen; 787 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 788 mpalen += sizeof(struct mpa_v2_conn_params); 789 wrlen = roundup(mpalen + sizeof *req, 16); 790 791 skb = get_skb(NULL, wrlen, GFP_KERNEL); 792 if (!skb) { 793 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 794 return -ENOMEM; 795 } 796 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 797 798 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 799 memset(req, 0, wrlen); 800 req->op_to_immdlen = cpu_to_be32( 801 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 802 FW_WR_COMPL(1) | 803 FW_WR_IMMDLEN(mpalen)); 804 req->flowid_len16 = cpu_to_be32( 805 FW_WR_FLOWID(ep->hwtid) | 806 FW_WR_LEN16(wrlen >> 4)); 807 req->plen = cpu_to_be32(mpalen); 808 req->tunnel_to_proxy = cpu_to_be32( 809 FW_OFLD_TX_DATA_WR_FLUSH(1) | 810 FW_OFLD_TX_DATA_WR_SHOVE(1)); 811 812 mpa = (struct mpa_message *)(req + 1); 813 memset(mpa, 0, sizeof(*mpa)); 814 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 815 mpa->flags = MPA_REJECT; 816 mpa->revision = ep->mpa_attr.version; 817 mpa->private_data_size = htons(plen); 818 819 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 820 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 821 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 822 sizeof (struct mpa_v2_conn_params)); 823 mpa_v2_params.ird = htons(((u16)ep->ird) | 824 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 825 0)); 826 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 827 (p2p_type == 828 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 829 MPA_V2_RDMA_WRITE_RTR : p2p_type == 830 FW_RI_INIT_P2PTYPE_READ_REQ ? 831 MPA_V2_RDMA_READ_RTR : 0) : 0)); 832 memcpy(mpa->private_data, &mpa_v2_params, 833 sizeof(struct mpa_v2_conn_params)); 834 835 if (ep->plen) 836 memcpy(mpa->private_data + 837 sizeof(struct mpa_v2_conn_params), pdata, plen); 838 } else 839 if (plen) 840 memcpy(mpa->private_data, pdata, plen); 841 842 /* 843 * Reference the mpa skb again. This ensures the data area 844 * will remain in memory until the hw acks the tx. 845 * Function fw4_ack() will deref it. 846 */ 847 skb_get(skb); 848 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 849 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 850 BUG_ON(ep->mpa_skb); 851 ep->mpa_skb = skb; 852 ep->snd_seq += mpalen; 853 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 854 } 855 856 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 857 { 858 int mpalen, wrlen; 859 struct fw_ofld_tx_data_wr *req; 860 struct mpa_message *mpa; 861 struct sk_buff *skb; 862 struct mpa_v2_conn_params mpa_v2_params; 863 864 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 865 866 mpalen = sizeof(*mpa) + plen; 867 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 868 mpalen += sizeof(struct mpa_v2_conn_params); 869 wrlen = roundup(mpalen + sizeof *req, 16); 870 871 skb = get_skb(NULL, wrlen, GFP_KERNEL); 872 if (!skb) { 873 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 874 return -ENOMEM; 875 } 876 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 877 878 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 879 memset(req, 0, wrlen); 880 req->op_to_immdlen = cpu_to_be32( 881 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 882 FW_WR_COMPL(1) | 883 FW_WR_IMMDLEN(mpalen)); 884 req->flowid_len16 = cpu_to_be32( 885 FW_WR_FLOWID(ep->hwtid) | 886 FW_WR_LEN16(wrlen >> 4)); 887 req->plen = cpu_to_be32(mpalen); 888 req->tunnel_to_proxy = cpu_to_be32( 889 FW_OFLD_TX_DATA_WR_FLUSH(1) | 890 FW_OFLD_TX_DATA_WR_SHOVE(1)); 891 892 mpa = (struct mpa_message *)(req + 1); 893 memset(mpa, 0, sizeof(*mpa)); 894 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 895 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 896 (markers_enabled ? MPA_MARKERS : 0); 897 mpa->revision = ep->mpa_attr.version; 898 mpa->private_data_size = htons(plen); 899 900 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 901 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 902 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 903 sizeof (struct mpa_v2_conn_params)); 904 mpa_v2_params.ird = htons((u16)ep->ird); 905 mpa_v2_params.ord = htons((u16)ep->ord); 906 if (peer2peer && (ep->mpa_attr.p2p_type != 907 FW_RI_INIT_P2PTYPE_DISABLED)) { 908 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 909 910 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 911 mpa_v2_params.ord |= 912 htons(MPA_V2_RDMA_WRITE_RTR); 913 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 914 mpa_v2_params.ord |= 915 htons(MPA_V2_RDMA_READ_RTR); 916 } 917 918 memcpy(mpa->private_data, &mpa_v2_params, 919 sizeof(struct mpa_v2_conn_params)); 920 921 if (ep->plen) 922 memcpy(mpa->private_data + 923 sizeof(struct mpa_v2_conn_params), pdata, plen); 924 } else 925 if (plen) 926 memcpy(mpa->private_data, pdata, plen); 927 928 /* 929 * Reference the mpa skb. This ensures the data area 930 * will remain in memory until the hw acks the tx. 931 * Function fw4_ack() will deref it. 932 */ 933 skb_get(skb); 934 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 935 ep->mpa_skb = skb; 936 __state_set(&ep->com, MPA_REP_SENT); 937 ep->snd_seq += mpalen; 938 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 939 } 940 941 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 942 { 943 struct c4iw_ep *ep; 944 struct cpl_act_establish *req = cplhdr(skb); 945 unsigned int tid = GET_TID(req); 946 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 947 struct tid_info *t = dev->rdev.lldi.tids; 948 949 ep = lookup_atid(t, atid); 950 951 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 952 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 953 954 mutex_lock(&ep->com.mutex); 955 dst_confirm(ep->dst); 956 957 /* setup the hwtid for this connection */ 958 ep->hwtid = tid; 959 cxgb4_insert_tid(t, ep, tid); 960 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 961 962 ep->snd_seq = be32_to_cpu(req->snd_isn); 963 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 964 965 set_emss(ep, ntohs(req->tcp_opt)); 966 967 /* dealloc the atid */ 968 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 969 cxgb4_free_atid(t, atid); 970 set_bit(ACT_ESTAB, &ep->com.history); 971 972 /* start MPA negotiation */ 973 send_flowc(ep, NULL); 974 if (ep->retry_with_mpa_v1) 975 send_mpa_req(ep, skb, 1); 976 else 977 send_mpa_req(ep, skb, mpa_rev); 978 mutex_unlock(&ep->com.mutex); 979 return 0; 980 } 981 982 static void close_complete_upcall(struct c4iw_ep *ep, int status) 983 { 984 struct iw_cm_event event; 985 986 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 987 memset(&event, 0, sizeof(event)); 988 event.event = IW_CM_EVENT_CLOSE; 989 event.status = status; 990 if (ep->com.cm_id) { 991 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 992 ep, ep->com.cm_id, ep->hwtid); 993 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 994 ep->com.cm_id->rem_ref(ep->com.cm_id); 995 ep->com.cm_id = NULL; 996 set_bit(CLOSE_UPCALL, &ep->com.history); 997 } 998 } 999 1000 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 1001 { 1002 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1003 __state_set(&ep->com, ABORTING); 1004 set_bit(ABORT_CONN, &ep->com.history); 1005 return send_abort(ep, skb, gfp); 1006 } 1007 1008 static void peer_close_upcall(struct c4iw_ep *ep) 1009 { 1010 struct iw_cm_event event; 1011 1012 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1013 memset(&event, 0, sizeof(event)); 1014 event.event = IW_CM_EVENT_DISCONNECT; 1015 if (ep->com.cm_id) { 1016 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 1017 ep, ep->com.cm_id, ep->hwtid); 1018 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1019 set_bit(DISCONN_UPCALL, &ep->com.history); 1020 } 1021 } 1022 1023 static void peer_abort_upcall(struct c4iw_ep *ep) 1024 { 1025 struct iw_cm_event event; 1026 1027 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1028 memset(&event, 0, sizeof(event)); 1029 event.event = IW_CM_EVENT_CLOSE; 1030 event.status = -ECONNRESET; 1031 if (ep->com.cm_id) { 1032 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 1033 ep->com.cm_id, ep->hwtid); 1034 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1035 ep->com.cm_id->rem_ref(ep->com.cm_id); 1036 ep->com.cm_id = NULL; 1037 set_bit(ABORT_UPCALL, &ep->com.history); 1038 } 1039 } 1040 1041 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1042 { 1043 struct iw_cm_event event; 1044 1045 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 1046 memset(&event, 0, sizeof(event)); 1047 event.event = IW_CM_EVENT_CONNECT_REPLY; 1048 event.status = status; 1049 memcpy(&event.local_addr, &ep->com.local_addr, 1050 sizeof(ep->com.local_addr)); 1051 memcpy(&event.remote_addr, &ep->com.remote_addr, 1052 sizeof(ep->com.remote_addr)); 1053 1054 if ((status == 0) || (status == -ECONNREFUSED)) { 1055 if (!ep->tried_with_mpa_v1) { 1056 /* this means MPA_v2 is used */ 1057 event.private_data_len = ep->plen - 1058 sizeof(struct mpa_v2_conn_params); 1059 event.private_data = ep->mpa_pkt + 1060 sizeof(struct mpa_message) + 1061 sizeof(struct mpa_v2_conn_params); 1062 } else { 1063 /* this means MPA_v1 is used */ 1064 event.private_data_len = ep->plen; 1065 event.private_data = ep->mpa_pkt + 1066 sizeof(struct mpa_message); 1067 } 1068 } 1069 1070 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 1071 ep->hwtid, status); 1072 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1073 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1074 1075 if (status < 0) { 1076 ep->com.cm_id->rem_ref(ep->com.cm_id); 1077 ep->com.cm_id = NULL; 1078 } 1079 } 1080 1081 static int connect_request_upcall(struct c4iw_ep *ep) 1082 { 1083 struct iw_cm_event event; 1084 int ret; 1085 1086 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1087 memset(&event, 0, sizeof(event)); 1088 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1089 memcpy(&event.local_addr, &ep->com.local_addr, 1090 sizeof(ep->com.local_addr)); 1091 memcpy(&event.remote_addr, &ep->com.remote_addr, 1092 sizeof(ep->com.remote_addr)); 1093 event.provider_data = ep; 1094 if (!ep->tried_with_mpa_v1) { 1095 /* this means MPA_v2 is used */ 1096 event.ord = ep->ord; 1097 event.ird = ep->ird; 1098 event.private_data_len = ep->plen - 1099 sizeof(struct mpa_v2_conn_params); 1100 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1101 sizeof(struct mpa_v2_conn_params); 1102 } else { 1103 /* this means MPA_v1 is used. Send max supported */ 1104 event.ord = c4iw_max_read_depth; 1105 event.ird = c4iw_max_read_depth; 1106 event.private_data_len = ep->plen; 1107 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1108 } 1109 c4iw_get_ep(&ep->com); 1110 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1111 &event); 1112 if (ret) 1113 c4iw_put_ep(&ep->com); 1114 set_bit(CONNREQ_UPCALL, &ep->com.history); 1115 c4iw_put_ep(&ep->parent_ep->com); 1116 return ret; 1117 } 1118 1119 static void established_upcall(struct c4iw_ep *ep) 1120 { 1121 struct iw_cm_event event; 1122 1123 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1124 memset(&event, 0, sizeof(event)); 1125 event.event = IW_CM_EVENT_ESTABLISHED; 1126 event.ird = ep->ird; 1127 event.ord = ep->ord; 1128 if (ep->com.cm_id) { 1129 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1130 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1131 set_bit(ESTAB_UPCALL, &ep->com.history); 1132 } 1133 } 1134 1135 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 1136 { 1137 struct cpl_rx_data_ack *req; 1138 struct sk_buff *skb; 1139 int wrlen = roundup(sizeof *req, 16); 1140 1141 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 1142 skb = get_skb(NULL, wrlen, GFP_KERNEL); 1143 if (!skb) { 1144 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 1145 return 0; 1146 } 1147 1148 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 1149 memset(req, 0, wrlen); 1150 INIT_TP_WR(req, ep->hwtid); 1151 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1152 ep->hwtid)); 1153 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 1154 F_RX_DACK_CHANGE | 1155 V_RX_DACK_MODE(dack_mode)); 1156 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 1157 c4iw_ofld_send(&ep->com.dev->rdev, skb); 1158 return credits; 1159 } 1160 1161 static int process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 1162 { 1163 struct mpa_message *mpa; 1164 struct mpa_v2_conn_params *mpa_v2_params; 1165 u16 plen; 1166 u16 resp_ird, resp_ord; 1167 u8 rtr_mismatch = 0, insuff_ird = 0; 1168 struct c4iw_qp_attributes attrs; 1169 enum c4iw_qp_attr_mask mask; 1170 int err; 1171 int disconnect = 0; 1172 1173 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1174 1175 /* 1176 * Stop mpa timer. If it expired, then 1177 * we ignore the MPA reply. process_timeout() 1178 * will abort the connection. 1179 */ 1180 if (stop_ep_timer(ep)) 1181 return 0; 1182 1183 /* 1184 * If we get more than the supported amount of private data 1185 * then we must fail this connection. 1186 */ 1187 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1188 err = -EINVAL; 1189 goto err; 1190 } 1191 1192 /* 1193 * copy the new data into our accumulation buffer. 1194 */ 1195 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1196 skb->len); 1197 ep->mpa_pkt_len += skb->len; 1198 1199 /* 1200 * if we don't even have the mpa message, then bail. 1201 */ 1202 if (ep->mpa_pkt_len < sizeof(*mpa)) 1203 return 0; 1204 mpa = (struct mpa_message *) ep->mpa_pkt; 1205 1206 /* Validate MPA header. */ 1207 if (mpa->revision > mpa_rev) { 1208 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1209 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1210 err = -EPROTO; 1211 goto err; 1212 } 1213 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1214 err = -EPROTO; 1215 goto err; 1216 } 1217 1218 plen = ntohs(mpa->private_data_size); 1219 1220 /* 1221 * Fail if there's too much private data. 1222 */ 1223 if (plen > MPA_MAX_PRIVATE_DATA) { 1224 err = -EPROTO; 1225 goto err; 1226 } 1227 1228 /* 1229 * If plen does not account for pkt size 1230 */ 1231 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1232 err = -EPROTO; 1233 goto err; 1234 } 1235 1236 ep->plen = (u8) plen; 1237 1238 /* 1239 * If we don't have all the pdata yet, then bail. 1240 * We'll continue process when more data arrives. 1241 */ 1242 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1243 return 0; 1244 1245 if (mpa->flags & MPA_REJECT) { 1246 err = -ECONNREFUSED; 1247 goto err; 1248 } 1249 1250 /* 1251 * If we get here we have accumulated the entire mpa 1252 * start reply message including private data. And 1253 * the MPA header is valid. 1254 */ 1255 __state_set(&ep->com, FPDU_MODE); 1256 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1257 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1258 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1259 ep->mpa_attr.version = mpa->revision; 1260 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1261 1262 if (mpa->revision == 2) { 1263 ep->mpa_attr.enhanced_rdma_conn = 1264 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1265 if (ep->mpa_attr.enhanced_rdma_conn) { 1266 mpa_v2_params = (struct mpa_v2_conn_params *) 1267 (ep->mpa_pkt + sizeof(*mpa)); 1268 resp_ird = ntohs(mpa_v2_params->ird) & 1269 MPA_V2_IRD_ORD_MASK; 1270 resp_ord = ntohs(mpa_v2_params->ord) & 1271 MPA_V2_IRD_ORD_MASK; 1272 1273 /* 1274 * This is a double-check. Ideally, below checks are 1275 * not required since ird/ord stuff has been taken 1276 * care of in c4iw_accept_cr 1277 */ 1278 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1279 err = -ENOMEM; 1280 ep->ird = resp_ord; 1281 ep->ord = resp_ird; 1282 insuff_ird = 1; 1283 } 1284 1285 if (ntohs(mpa_v2_params->ird) & 1286 MPA_V2_PEER2PEER_MODEL) { 1287 if (ntohs(mpa_v2_params->ord) & 1288 MPA_V2_RDMA_WRITE_RTR) 1289 ep->mpa_attr.p2p_type = 1290 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1291 else if (ntohs(mpa_v2_params->ord) & 1292 MPA_V2_RDMA_READ_RTR) 1293 ep->mpa_attr.p2p_type = 1294 FW_RI_INIT_P2PTYPE_READ_REQ; 1295 } 1296 } 1297 } else if (mpa->revision == 1) 1298 if (peer2peer) 1299 ep->mpa_attr.p2p_type = p2p_type; 1300 1301 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1302 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1303 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1304 ep->mpa_attr.recv_marker_enabled, 1305 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1306 ep->mpa_attr.p2p_type, p2p_type); 1307 1308 /* 1309 * If responder's RTR does not match with that of initiator, assign 1310 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1311 * generated when moving QP to RTS state. 1312 * A TERM message will be sent after QP has moved to RTS state 1313 */ 1314 if ((ep->mpa_attr.version == 2) && peer2peer && 1315 (ep->mpa_attr.p2p_type != p2p_type)) { 1316 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1317 rtr_mismatch = 1; 1318 } 1319 1320 attrs.mpa_attr = ep->mpa_attr; 1321 attrs.max_ird = ep->ird; 1322 attrs.max_ord = ep->ord; 1323 attrs.llp_stream_handle = ep; 1324 attrs.next_state = C4IW_QP_STATE_RTS; 1325 1326 mask = C4IW_QP_ATTR_NEXT_STATE | 1327 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1328 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1329 1330 /* bind QP and TID with INIT_WR */ 1331 err = c4iw_modify_qp(ep->com.qp->rhp, 1332 ep->com.qp, mask, &attrs, 1); 1333 if (err) 1334 goto err; 1335 1336 /* 1337 * If responder's RTR requirement did not match with what initiator 1338 * supports, generate TERM message 1339 */ 1340 if (rtr_mismatch) { 1341 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1342 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1343 attrs.ecode = MPA_NOMATCH_RTR; 1344 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1345 attrs.send_term = 1; 1346 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1347 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1348 err = -ENOMEM; 1349 disconnect = 1; 1350 goto out; 1351 } 1352 1353 /* 1354 * Generate TERM if initiator IRD is not sufficient for responder 1355 * provided ORD. Currently, we do the same behaviour even when 1356 * responder provided IRD is also not sufficient as regards to 1357 * initiator ORD. 1358 */ 1359 if (insuff_ird) { 1360 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1361 __func__); 1362 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1363 attrs.ecode = MPA_INSUFF_IRD; 1364 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1365 attrs.send_term = 1; 1366 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1367 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1368 err = -ENOMEM; 1369 disconnect = 1; 1370 goto out; 1371 } 1372 goto out; 1373 err: 1374 __state_set(&ep->com, ABORTING); 1375 send_abort(ep, skb, GFP_KERNEL); 1376 out: 1377 connect_reply_upcall(ep, err); 1378 return disconnect; 1379 } 1380 1381 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1382 { 1383 struct mpa_message *mpa; 1384 struct mpa_v2_conn_params *mpa_v2_params; 1385 u16 plen; 1386 1387 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1388 1389 /* 1390 * If we get more than the supported amount of private data 1391 * then we must fail this connection. 1392 */ 1393 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1394 (void)stop_ep_timer(ep); 1395 abort_connection(ep, skb, GFP_KERNEL); 1396 return; 1397 } 1398 1399 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1400 1401 /* 1402 * Copy the new data into our accumulation buffer. 1403 */ 1404 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1405 skb->len); 1406 ep->mpa_pkt_len += skb->len; 1407 1408 /* 1409 * If we don't even have the mpa message, then bail. 1410 * We'll continue process when more data arrives. 1411 */ 1412 if (ep->mpa_pkt_len < sizeof(*mpa)) 1413 return; 1414 1415 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1416 mpa = (struct mpa_message *) ep->mpa_pkt; 1417 1418 /* 1419 * Validate MPA Header. 1420 */ 1421 if (mpa->revision > mpa_rev) { 1422 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1423 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1424 (void)stop_ep_timer(ep); 1425 abort_connection(ep, skb, GFP_KERNEL); 1426 return; 1427 } 1428 1429 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1430 (void)stop_ep_timer(ep); 1431 abort_connection(ep, skb, GFP_KERNEL); 1432 return; 1433 } 1434 1435 plen = ntohs(mpa->private_data_size); 1436 1437 /* 1438 * Fail if there's too much private data. 1439 */ 1440 if (plen > MPA_MAX_PRIVATE_DATA) { 1441 (void)stop_ep_timer(ep); 1442 abort_connection(ep, skb, GFP_KERNEL); 1443 return; 1444 } 1445 1446 /* 1447 * If plen does not account for pkt size 1448 */ 1449 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1450 (void)stop_ep_timer(ep); 1451 abort_connection(ep, skb, GFP_KERNEL); 1452 return; 1453 } 1454 ep->plen = (u8) plen; 1455 1456 /* 1457 * If we don't have all the pdata yet, then bail. 1458 */ 1459 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1460 return; 1461 1462 /* 1463 * If we get here we have accumulated the entire mpa 1464 * start reply message including private data. 1465 */ 1466 ep->mpa_attr.initiator = 0; 1467 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1468 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1469 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1470 ep->mpa_attr.version = mpa->revision; 1471 if (mpa->revision == 1) 1472 ep->tried_with_mpa_v1 = 1; 1473 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1474 1475 if (mpa->revision == 2) { 1476 ep->mpa_attr.enhanced_rdma_conn = 1477 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1478 if (ep->mpa_attr.enhanced_rdma_conn) { 1479 mpa_v2_params = (struct mpa_v2_conn_params *) 1480 (ep->mpa_pkt + sizeof(*mpa)); 1481 ep->ird = ntohs(mpa_v2_params->ird) & 1482 MPA_V2_IRD_ORD_MASK; 1483 ep->ord = ntohs(mpa_v2_params->ord) & 1484 MPA_V2_IRD_ORD_MASK; 1485 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1486 if (peer2peer) { 1487 if (ntohs(mpa_v2_params->ord) & 1488 MPA_V2_RDMA_WRITE_RTR) 1489 ep->mpa_attr.p2p_type = 1490 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1491 else if (ntohs(mpa_v2_params->ord) & 1492 MPA_V2_RDMA_READ_RTR) 1493 ep->mpa_attr.p2p_type = 1494 FW_RI_INIT_P2PTYPE_READ_REQ; 1495 } 1496 } 1497 } else if (mpa->revision == 1) 1498 if (peer2peer) 1499 ep->mpa_attr.p2p_type = p2p_type; 1500 1501 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1502 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1503 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1504 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1505 ep->mpa_attr.p2p_type); 1506 1507 /* 1508 * If the endpoint timer already expired, then we ignore 1509 * the start request. process_timeout() will abort 1510 * the connection. 1511 */ 1512 if (!stop_ep_timer(ep)) { 1513 __state_set(&ep->com, MPA_REQ_RCVD); 1514 1515 /* drive upcall */ 1516 mutex_lock(&ep->parent_ep->com.mutex); 1517 if (ep->parent_ep->com.state != DEAD) { 1518 if (connect_request_upcall(ep)) 1519 abort_connection(ep, skb, GFP_KERNEL); 1520 } else { 1521 abort_connection(ep, skb, GFP_KERNEL); 1522 } 1523 mutex_unlock(&ep->parent_ep->com.mutex); 1524 } 1525 return; 1526 } 1527 1528 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1529 { 1530 struct c4iw_ep *ep; 1531 struct cpl_rx_data *hdr = cplhdr(skb); 1532 unsigned int dlen = ntohs(hdr->len); 1533 unsigned int tid = GET_TID(hdr); 1534 struct tid_info *t = dev->rdev.lldi.tids; 1535 __u8 status = hdr->status; 1536 int disconnect = 0; 1537 1538 ep = lookup_tid(t, tid); 1539 if (!ep) 1540 return 0; 1541 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1542 skb_pull(skb, sizeof(*hdr)); 1543 skb_trim(skb, dlen); 1544 mutex_lock(&ep->com.mutex); 1545 1546 /* update RX credits */ 1547 update_rx_credits(ep, dlen); 1548 1549 switch (ep->com.state) { 1550 case MPA_REQ_SENT: 1551 ep->rcv_seq += dlen; 1552 disconnect = process_mpa_reply(ep, skb); 1553 break; 1554 case MPA_REQ_WAIT: 1555 ep->rcv_seq += dlen; 1556 process_mpa_request(ep, skb); 1557 break; 1558 case FPDU_MODE: { 1559 struct c4iw_qp_attributes attrs; 1560 BUG_ON(!ep->com.qp); 1561 if (status) 1562 pr_err("%s Unexpected streaming data." \ 1563 " qpid %u ep %p state %d tid %u status %d\n", 1564 __func__, ep->com.qp->wq.sq.qid, ep, 1565 ep->com.state, ep->hwtid, status); 1566 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1567 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1568 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1569 disconnect = 1; 1570 break; 1571 } 1572 default: 1573 break; 1574 } 1575 mutex_unlock(&ep->com.mutex); 1576 if (disconnect) 1577 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 1578 return 0; 1579 } 1580 1581 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1582 { 1583 struct c4iw_ep *ep; 1584 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1585 int release = 0; 1586 unsigned int tid = GET_TID(rpl); 1587 struct tid_info *t = dev->rdev.lldi.tids; 1588 1589 ep = lookup_tid(t, tid); 1590 if (!ep) { 1591 printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n"); 1592 return 0; 1593 } 1594 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1595 mutex_lock(&ep->com.mutex); 1596 switch (ep->com.state) { 1597 case ABORTING: 1598 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1599 __state_set(&ep->com, DEAD); 1600 release = 1; 1601 break; 1602 default: 1603 printk(KERN_ERR "%s ep %p state %d\n", 1604 __func__, ep, ep->com.state); 1605 break; 1606 } 1607 mutex_unlock(&ep->com.mutex); 1608 1609 if (release) 1610 release_ep_resources(ep); 1611 return 0; 1612 } 1613 1614 static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid) 1615 { 1616 struct sk_buff *skb; 1617 struct fw_ofld_connection_wr *req; 1618 unsigned int mtu_idx; 1619 int wscale; 1620 struct sockaddr_in *sin; 1621 1622 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1623 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1624 memset(req, 0, sizeof(*req)); 1625 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR)); 1626 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1627 req->le.filter = cpu_to_be32(cxgb4_select_ntuple( 1628 ep->com.dev->rdev.lldi.ports[0], 1629 ep->l2t)); 1630 sin = (struct sockaddr_in *)&ep->com.local_addr; 1631 req->le.lport = sin->sin_port; 1632 req->le.u.ipv4.lip = sin->sin_addr.s_addr; 1633 sin = (struct sockaddr_in *)&ep->com.remote_addr; 1634 req->le.pport = sin->sin_port; 1635 req->le.u.ipv4.pip = sin->sin_addr.s_addr; 1636 req->tcb.t_state_to_astid = 1637 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1638 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1639 req->tcb.cplrxdataack_cplpassacceptrpl = 1640 htons(F_FW_OFLD_CONNECTION_WR_CPLRXDATAACK); 1641 req->tcb.tx_max = (__force __be32) jiffies; 1642 req->tcb.rcv_adv = htons(1); 1643 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1644 wscale = compute_wscale(rcv_win); 1645 req->tcb.opt0 = (__force __be64) (TCAM_BYPASS(1) | 1646 (nocong ? NO_CONG(1) : 0) | 1647 KEEP_ALIVE(1) | 1648 DELACK(1) | 1649 WND_SCALE(wscale) | 1650 MSS_IDX(mtu_idx) | 1651 L2T_IDX(ep->l2t->idx) | 1652 TX_CHAN(ep->tx_chan) | 1653 SMAC_SEL(ep->smac_idx) | 1654 DSCP(ep->tos) | 1655 ULP_MODE(ULP_MODE_TCPDDP) | 1656 RCV_BUFSIZ(rcv_win >> 10)); 1657 req->tcb.opt2 = (__force __be32) (PACE(1) | 1658 TX_QUEUE(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) | 1659 RX_CHANNEL(0) | 1660 CCTRL_ECN(enable_ecn) | 1661 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid)); 1662 if (enable_tcp_timestamps) 1663 req->tcb.opt2 |= (__force __be32) TSTAMPS_EN(1); 1664 if (enable_tcp_sack) 1665 req->tcb.opt2 |= (__force __be32) SACK_EN(1); 1666 if (wscale && enable_tcp_window_scaling) 1667 req->tcb.opt2 |= (__force __be32) WND_SCALE_EN(1); 1668 req->tcb.opt0 = cpu_to_be64((__force u64) req->tcb.opt0); 1669 req->tcb.opt2 = cpu_to_be32((__force u32) req->tcb.opt2); 1670 set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx); 1671 set_bit(ACT_OFLD_CONN, &ep->com.history); 1672 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1673 } 1674 1675 /* 1676 * Return whether a failed active open has allocated a TID 1677 */ 1678 static inline int act_open_has_tid(int status) 1679 { 1680 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1681 status != CPL_ERR_ARP_MISS; 1682 } 1683 1684 /* Returns whether a CPL status conveys negative advice. 1685 */ 1686 static int is_neg_adv(unsigned int status) 1687 { 1688 return status == CPL_ERR_RTX_NEG_ADVICE || 1689 status == CPL_ERR_PERSIST_NEG_ADVICE || 1690 status == CPL_ERR_KEEPALV_NEG_ADVICE; 1691 } 1692 1693 #define ACT_OPEN_RETRY_COUNT 2 1694 1695 static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip, 1696 struct dst_entry *dst, struct c4iw_dev *cdev, 1697 bool clear_mpa_v1) 1698 { 1699 struct neighbour *n; 1700 int err, step; 1701 struct net_device *pdev; 1702 1703 n = dst_neigh_lookup(dst, peer_ip); 1704 if (!n) 1705 return -ENODEV; 1706 1707 rcu_read_lock(); 1708 err = -ENOMEM; 1709 if (n->dev->flags & IFF_LOOPBACK) { 1710 if (iptype == 4) 1711 pdev = ip_dev_find(&init_net, *(__be32 *)peer_ip); 1712 else if (IS_ENABLED(CONFIG_IPV6)) 1713 for_each_netdev(&init_net, pdev) { 1714 if (ipv6_chk_addr(&init_net, 1715 (struct in6_addr *)peer_ip, 1716 pdev, 1)) 1717 break; 1718 } 1719 else 1720 pdev = NULL; 1721 1722 if (!pdev) { 1723 err = -ENODEV; 1724 goto out; 1725 } 1726 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1727 n, pdev, 0); 1728 if (!ep->l2t) 1729 goto out; 1730 ep->mtu = pdev->mtu; 1731 ep->tx_chan = cxgb4_port_chan(pdev); 1732 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1733 step = cdev->rdev.lldi.ntxq / 1734 cdev->rdev.lldi.nchan; 1735 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1736 step = cdev->rdev.lldi.nrxq / 1737 cdev->rdev.lldi.nchan; 1738 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1739 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1740 cxgb4_port_idx(pdev) * step]; 1741 dev_put(pdev); 1742 } else { 1743 pdev = get_real_dev(n->dev); 1744 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1745 n, pdev, 0); 1746 if (!ep->l2t) 1747 goto out; 1748 ep->mtu = dst_mtu(dst); 1749 ep->tx_chan = cxgb4_port_chan(n->dev); 1750 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1751 step = cdev->rdev.lldi.ntxq / 1752 cdev->rdev.lldi.nchan; 1753 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1754 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1755 step = cdev->rdev.lldi.nrxq / 1756 cdev->rdev.lldi.nchan; 1757 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1758 cxgb4_port_idx(n->dev) * step]; 1759 1760 if (clear_mpa_v1) { 1761 ep->retry_with_mpa_v1 = 0; 1762 ep->tried_with_mpa_v1 = 0; 1763 } 1764 } 1765 err = 0; 1766 out: 1767 rcu_read_unlock(); 1768 1769 neigh_release(n); 1770 1771 return err; 1772 } 1773 1774 static int c4iw_reconnect(struct c4iw_ep *ep) 1775 { 1776 int err = 0; 1777 struct sockaddr_in *laddr = (struct sockaddr_in *) 1778 &ep->com.cm_id->local_addr; 1779 struct sockaddr_in *raddr = (struct sockaddr_in *) 1780 &ep->com.cm_id->remote_addr; 1781 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *) 1782 &ep->com.cm_id->local_addr; 1783 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 1784 &ep->com.cm_id->remote_addr; 1785 int iptype; 1786 __u8 *ra; 1787 1788 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1789 init_timer(&ep->timer); 1790 1791 /* 1792 * Allocate an active TID to initiate a TCP connection. 1793 */ 1794 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1795 if (ep->atid == -1) { 1796 pr_err("%s - cannot alloc atid.\n", __func__); 1797 err = -ENOMEM; 1798 goto fail2; 1799 } 1800 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1801 1802 /* find a route */ 1803 if (ep->com.cm_id->local_addr.ss_family == AF_INET) { 1804 ep->dst = find_route(ep->com.dev, laddr->sin_addr.s_addr, 1805 raddr->sin_addr.s_addr, laddr->sin_port, 1806 raddr->sin_port, 0); 1807 iptype = 4; 1808 ra = (__u8 *)&raddr->sin_addr; 1809 } else { 1810 ep->dst = find_route6(ep->com.dev, laddr6->sin6_addr.s6_addr, 1811 raddr6->sin6_addr.s6_addr, 1812 laddr6->sin6_port, raddr6->sin6_port, 0, 1813 raddr6->sin6_scope_id); 1814 iptype = 6; 1815 ra = (__u8 *)&raddr6->sin6_addr; 1816 } 1817 if (!ep->dst) { 1818 pr_err("%s - cannot find route.\n", __func__); 1819 err = -EHOSTUNREACH; 1820 goto fail3; 1821 } 1822 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, false); 1823 if (err) { 1824 pr_err("%s - cannot alloc l2e.\n", __func__); 1825 goto fail4; 1826 } 1827 1828 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1829 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1830 ep->l2t->idx); 1831 1832 state_set(&ep->com, CONNECTING); 1833 ep->tos = 0; 1834 1835 /* send connect request to rnic */ 1836 err = send_connect(ep); 1837 if (!err) 1838 goto out; 1839 1840 cxgb4_l2t_release(ep->l2t); 1841 fail4: 1842 dst_release(ep->dst); 1843 fail3: 1844 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 1845 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1846 fail2: 1847 /* 1848 * remember to send notification to upper layer. 1849 * We are in here so the upper layer is not aware that this is 1850 * re-connect attempt and so, upper layer is still waiting for 1851 * response of 1st connect request. 1852 */ 1853 connect_reply_upcall(ep, -ECONNRESET); 1854 c4iw_put_ep(&ep->com); 1855 out: 1856 return err; 1857 } 1858 1859 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1860 { 1861 struct c4iw_ep *ep; 1862 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1863 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1864 ntohl(rpl->atid_status))); 1865 struct tid_info *t = dev->rdev.lldi.tids; 1866 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1867 struct sockaddr_in *la; 1868 struct sockaddr_in *ra; 1869 struct sockaddr_in6 *la6; 1870 struct sockaddr_in6 *ra6; 1871 1872 ep = lookup_atid(t, atid); 1873 la = (struct sockaddr_in *)&ep->com.local_addr; 1874 ra = (struct sockaddr_in *)&ep->com.remote_addr; 1875 la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 1876 ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 1877 1878 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1879 status, status2errno(status)); 1880 1881 if (is_neg_adv(status)) { 1882 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1883 atid); 1884 return 0; 1885 } 1886 1887 set_bit(ACT_OPEN_RPL, &ep->com.history); 1888 1889 /* 1890 * Log interesting failures. 1891 */ 1892 switch (status) { 1893 case CPL_ERR_CONN_RESET: 1894 case CPL_ERR_CONN_TIMEDOUT: 1895 break; 1896 case CPL_ERR_TCAM_FULL: 1897 mutex_lock(&dev->rdev.stats.lock); 1898 dev->rdev.stats.tcam_full++; 1899 mutex_unlock(&dev->rdev.stats.lock); 1900 if (ep->com.local_addr.ss_family == AF_INET && 1901 dev->rdev.lldi.enable_fw_ofld_conn) { 1902 send_fw_act_open_req(ep, 1903 GET_TID_TID(GET_AOPEN_ATID( 1904 ntohl(rpl->atid_status)))); 1905 return 0; 1906 } 1907 break; 1908 case CPL_ERR_CONN_EXIST: 1909 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 1910 set_bit(ACT_RETRY_INUSE, &ep->com.history); 1911 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, 1912 atid); 1913 cxgb4_free_atid(t, atid); 1914 dst_release(ep->dst); 1915 cxgb4_l2t_release(ep->l2t); 1916 c4iw_reconnect(ep); 1917 return 0; 1918 } 1919 break; 1920 default: 1921 if (ep->com.local_addr.ss_family == AF_INET) { 1922 pr_info("Active open failure - atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1923 atid, status, status2errno(status), 1924 &la->sin_addr.s_addr, ntohs(la->sin_port), 1925 &ra->sin_addr.s_addr, ntohs(ra->sin_port)); 1926 } else { 1927 pr_info("Active open failure - atid %u status %u errno %d %pI6:%u->%pI6:%u\n", 1928 atid, status, status2errno(status), 1929 la6->sin6_addr.s6_addr, ntohs(la6->sin6_port), 1930 ra6->sin6_addr.s6_addr, ntohs(ra6->sin6_port)); 1931 } 1932 break; 1933 } 1934 1935 connect_reply_upcall(ep, status2errno(status)); 1936 state_set(&ep->com, DEAD); 1937 1938 if (status && act_open_has_tid(status)) 1939 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1940 1941 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 1942 cxgb4_free_atid(t, atid); 1943 dst_release(ep->dst); 1944 cxgb4_l2t_release(ep->l2t); 1945 c4iw_put_ep(&ep->com); 1946 1947 return 0; 1948 } 1949 1950 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1951 { 1952 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1953 struct tid_info *t = dev->rdev.lldi.tids; 1954 unsigned int stid = GET_TID(rpl); 1955 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1956 1957 if (!ep) { 1958 PDBG("%s stid %d lookup failure!\n", __func__, stid); 1959 goto out; 1960 } 1961 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1962 rpl->status, status2errno(rpl->status)); 1963 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1964 1965 out: 1966 return 0; 1967 } 1968 1969 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1970 { 1971 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1972 struct tid_info *t = dev->rdev.lldi.tids; 1973 unsigned int stid = GET_TID(rpl); 1974 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1975 1976 PDBG("%s ep %p\n", __func__, ep); 1977 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1978 return 0; 1979 } 1980 1981 static void accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, 1982 struct cpl_pass_accept_req *req) 1983 { 1984 struct cpl_pass_accept_rpl *rpl; 1985 unsigned int mtu_idx; 1986 u64 opt0; 1987 u32 opt2; 1988 int wscale; 1989 1990 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1991 BUG_ON(skb_cloned(skb)); 1992 skb_trim(skb, sizeof(*rpl)); 1993 skb_get(skb); 1994 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1995 wscale = compute_wscale(rcv_win); 1996 opt0 = (nocong ? NO_CONG(1) : 0) | 1997 KEEP_ALIVE(1) | 1998 DELACK(1) | 1999 WND_SCALE(wscale) | 2000 MSS_IDX(mtu_idx) | 2001 L2T_IDX(ep->l2t->idx) | 2002 TX_CHAN(ep->tx_chan) | 2003 SMAC_SEL(ep->smac_idx) | 2004 DSCP(ep->tos >> 2) | 2005 ULP_MODE(ULP_MODE_TCPDDP) | 2006 RCV_BUFSIZ(rcv_win>>10); 2007 opt2 = RX_CHANNEL(0) | 2008 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 2009 2010 if (enable_tcp_timestamps && req->tcpopt.tstamp) 2011 opt2 |= TSTAMPS_EN(1); 2012 if (enable_tcp_sack && req->tcpopt.sack) 2013 opt2 |= SACK_EN(1); 2014 if (wscale && enable_tcp_window_scaling) 2015 opt2 |= WND_SCALE_EN(1); 2016 if (enable_ecn) { 2017 const struct tcphdr *tcph; 2018 u32 hlen = ntohl(req->hdr_len); 2019 2020 tcph = (const void *)(req + 1) + G_ETH_HDR_LEN(hlen) + 2021 G_IP_HDR_LEN(hlen); 2022 if (tcph->ece && tcph->cwr) 2023 opt2 |= CCTRL_ECN(1); 2024 } 2025 if (is_t5(ep->com.dev->rdev.lldi.adapter_type)) { 2026 opt2 |= T5_OPT_2_VALID; 2027 opt2 |= V_CONG_CNTRL(CONG_ALG_TAHOE); 2028 } 2029 2030 rpl = cplhdr(skb); 2031 INIT_TP_WR(rpl, ep->hwtid); 2032 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 2033 ep->hwtid)); 2034 rpl->opt0 = cpu_to_be64(opt0); 2035 rpl->opt2 = cpu_to_be32(opt2); 2036 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 2037 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 2038 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 2039 2040 return; 2041 } 2042 2043 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, struct sk_buff *skb) 2044 { 2045 PDBG("%s c4iw_dev %p tid %u\n", __func__, dev, hwtid); 2046 BUG_ON(skb_cloned(skb)); 2047 skb_trim(skb, sizeof(struct cpl_tid_release)); 2048 skb_get(skb); 2049 release_tid(&dev->rdev, hwtid, skb); 2050 return; 2051 } 2052 2053 static void get_4tuple(struct cpl_pass_accept_req *req, int *iptype, 2054 __u8 *local_ip, __u8 *peer_ip, 2055 __be16 *local_port, __be16 *peer_port) 2056 { 2057 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 2058 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 2059 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 2060 struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len); 2061 struct tcphdr *tcp = (struct tcphdr *) 2062 ((u8 *)(req + 1) + eth_len + ip_len); 2063 2064 if (ip->version == 4) { 2065 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 2066 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 2067 ntohs(tcp->dest)); 2068 *iptype = 4; 2069 memcpy(peer_ip, &ip->saddr, 4); 2070 memcpy(local_ip, &ip->daddr, 4); 2071 } else { 2072 PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__, 2073 ip6->saddr.s6_addr, ip6->daddr.s6_addr, ntohs(tcp->source), 2074 ntohs(tcp->dest)); 2075 *iptype = 6; 2076 memcpy(peer_ip, ip6->saddr.s6_addr, 16); 2077 memcpy(local_ip, ip6->daddr.s6_addr, 16); 2078 } 2079 *peer_port = tcp->source; 2080 *local_port = tcp->dest; 2081 2082 return; 2083 } 2084 2085 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 2086 { 2087 struct c4iw_ep *child_ep = NULL, *parent_ep; 2088 struct cpl_pass_accept_req *req = cplhdr(skb); 2089 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 2090 struct tid_info *t = dev->rdev.lldi.tids; 2091 unsigned int hwtid = GET_TID(req); 2092 struct dst_entry *dst; 2093 __u8 local_ip[16], peer_ip[16]; 2094 __be16 local_port, peer_port; 2095 int err; 2096 u16 peer_mss = ntohs(req->tcpopt.mss); 2097 int iptype; 2098 2099 parent_ep = lookup_stid(t, stid); 2100 if (!parent_ep) { 2101 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 2102 goto reject; 2103 } 2104 2105 if (state_read(&parent_ep->com) != LISTEN) { 2106 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 2107 __func__); 2108 goto reject; 2109 } 2110 2111 get_4tuple(req, &iptype, local_ip, peer_ip, &local_port, &peer_port); 2112 2113 /* Find output route */ 2114 if (iptype == 4) { 2115 PDBG("%s parent ep %p hwtid %u laddr %pI4 raddr %pI4 lport %d rport %d peer_mss %d\n" 2116 , __func__, parent_ep, hwtid, 2117 local_ip, peer_ip, ntohs(local_port), 2118 ntohs(peer_port), peer_mss); 2119 dst = find_route(dev, *(__be32 *)local_ip, *(__be32 *)peer_ip, 2120 local_port, peer_port, 2121 GET_POPEN_TOS(ntohl(req->tos_stid))); 2122 } else { 2123 PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n" 2124 , __func__, parent_ep, hwtid, 2125 local_ip, peer_ip, ntohs(local_port), 2126 ntohs(peer_port), peer_mss); 2127 dst = find_route6(dev, local_ip, peer_ip, local_port, peer_port, 2128 PASS_OPEN_TOS(ntohl(req->tos_stid)), 2129 ((struct sockaddr_in6 *) 2130 &parent_ep->com.local_addr)->sin6_scope_id); 2131 } 2132 if (!dst) { 2133 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 2134 __func__); 2135 goto reject; 2136 } 2137 2138 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 2139 if (!child_ep) { 2140 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 2141 __func__); 2142 dst_release(dst); 2143 goto reject; 2144 } 2145 2146 err = import_ep(child_ep, iptype, peer_ip, dst, dev, false); 2147 if (err) { 2148 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 2149 __func__); 2150 dst_release(dst); 2151 kfree(child_ep); 2152 goto reject; 2153 } 2154 2155 if (peer_mss && child_ep->mtu > (peer_mss + 40)) 2156 child_ep->mtu = peer_mss + 40; 2157 2158 state_set(&child_ep->com, CONNECTING); 2159 child_ep->com.dev = dev; 2160 child_ep->com.cm_id = NULL; 2161 if (iptype == 4) { 2162 struct sockaddr_in *sin = (struct sockaddr_in *) 2163 &child_ep->com.local_addr; 2164 sin->sin_family = PF_INET; 2165 sin->sin_port = local_port; 2166 sin->sin_addr.s_addr = *(__be32 *)local_ip; 2167 sin = (struct sockaddr_in *)&child_ep->com.remote_addr; 2168 sin->sin_family = PF_INET; 2169 sin->sin_port = peer_port; 2170 sin->sin_addr.s_addr = *(__be32 *)peer_ip; 2171 } else { 2172 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 2173 &child_ep->com.local_addr; 2174 sin6->sin6_family = PF_INET6; 2175 sin6->sin6_port = local_port; 2176 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16); 2177 sin6 = (struct sockaddr_in6 *)&child_ep->com.remote_addr; 2178 sin6->sin6_family = PF_INET6; 2179 sin6->sin6_port = peer_port; 2180 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16); 2181 } 2182 c4iw_get_ep(&parent_ep->com); 2183 child_ep->parent_ep = parent_ep; 2184 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 2185 child_ep->dst = dst; 2186 child_ep->hwtid = hwtid; 2187 2188 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 2189 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 2190 2191 init_timer(&child_ep->timer); 2192 cxgb4_insert_tid(t, child_ep, hwtid); 2193 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid); 2194 accept_cr(child_ep, skb, req); 2195 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 2196 goto out; 2197 reject: 2198 reject_cr(dev, hwtid, skb); 2199 out: 2200 return 0; 2201 } 2202 2203 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 2204 { 2205 struct c4iw_ep *ep; 2206 struct cpl_pass_establish *req = cplhdr(skb); 2207 struct tid_info *t = dev->rdev.lldi.tids; 2208 unsigned int tid = GET_TID(req); 2209 2210 ep = lookup_tid(t, tid); 2211 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2212 ep->snd_seq = be32_to_cpu(req->snd_isn); 2213 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 2214 2215 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__, ep, tid, 2216 ntohs(req->tcp_opt)); 2217 2218 set_emss(ep, ntohs(req->tcp_opt)); 2219 2220 dst_confirm(ep->dst); 2221 state_set(&ep->com, MPA_REQ_WAIT); 2222 start_ep_timer(ep); 2223 send_flowc(ep, skb); 2224 set_bit(PASS_ESTAB, &ep->com.history); 2225 2226 return 0; 2227 } 2228 2229 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 2230 { 2231 struct cpl_peer_close *hdr = cplhdr(skb); 2232 struct c4iw_ep *ep; 2233 struct c4iw_qp_attributes attrs; 2234 int disconnect = 1; 2235 int release = 0; 2236 struct tid_info *t = dev->rdev.lldi.tids; 2237 unsigned int tid = GET_TID(hdr); 2238 int ret; 2239 2240 ep = lookup_tid(t, tid); 2241 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2242 dst_confirm(ep->dst); 2243 2244 set_bit(PEER_CLOSE, &ep->com.history); 2245 mutex_lock(&ep->com.mutex); 2246 switch (ep->com.state) { 2247 case MPA_REQ_WAIT: 2248 __state_set(&ep->com, CLOSING); 2249 break; 2250 case MPA_REQ_SENT: 2251 __state_set(&ep->com, CLOSING); 2252 connect_reply_upcall(ep, -ECONNRESET); 2253 break; 2254 case MPA_REQ_RCVD: 2255 2256 /* 2257 * We're gonna mark this puppy DEAD, but keep 2258 * the reference on it until the ULP accepts or 2259 * rejects the CR. Also wake up anyone waiting 2260 * in rdma connection migration (see c4iw_accept_cr()). 2261 */ 2262 __state_set(&ep->com, CLOSING); 2263 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2264 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2265 break; 2266 case MPA_REP_SENT: 2267 __state_set(&ep->com, CLOSING); 2268 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2269 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2270 break; 2271 case FPDU_MODE: 2272 start_ep_timer(ep); 2273 __state_set(&ep->com, CLOSING); 2274 attrs.next_state = C4IW_QP_STATE_CLOSING; 2275 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2276 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2277 if (ret != -ECONNRESET) { 2278 peer_close_upcall(ep); 2279 disconnect = 1; 2280 } 2281 break; 2282 case ABORTING: 2283 disconnect = 0; 2284 break; 2285 case CLOSING: 2286 __state_set(&ep->com, MORIBUND); 2287 disconnect = 0; 2288 break; 2289 case MORIBUND: 2290 (void)stop_ep_timer(ep); 2291 if (ep->com.cm_id && ep->com.qp) { 2292 attrs.next_state = C4IW_QP_STATE_IDLE; 2293 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2294 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2295 } 2296 close_complete_upcall(ep, 0); 2297 __state_set(&ep->com, DEAD); 2298 release = 1; 2299 disconnect = 0; 2300 break; 2301 case DEAD: 2302 disconnect = 0; 2303 break; 2304 default: 2305 BUG_ON(1); 2306 } 2307 mutex_unlock(&ep->com.mutex); 2308 if (disconnect) 2309 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2310 if (release) 2311 release_ep_resources(ep); 2312 return 0; 2313 } 2314 2315 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 2316 { 2317 struct cpl_abort_req_rss *req = cplhdr(skb); 2318 struct c4iw_ep *ep; 2319 struct cpl_abort_rpl *rpl; 2320 struct sk_buff *rpl_skb; 2321 struct c4iw_qp_attributes attrs; 2322 int ret; 2323 int release = 0; 2324 struct tid_info *t = dev->rdev.lldi.tids; 2325 unsigned int tid = GET_TID(req); 2326 2327 ep = lookup_tid(t, tid); 2328 if (is_neg_adv(req->status)) { 2329 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2330 ep->hwtid); 2331 return 0; 2332 } 2333 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2334 ep->com.state); 2335 set_bit(PEER_ABORT, &ep->com.history); 2336 2337 /* 2338 * Wake up any threads in rdma_init() or rdma_fini(). 2339 * However, this is not needed if com state is just 2340 * MPA_REQ_SENT 2341 */ 2342 if (ep->com.state != MPA_REQ_SENT) 2343 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2344 2345 mutex_lock(&ep->com.mutex); 2346 switch (ep->com.state) { 2347 case CONNECTING: 2348 break; 2349 case MPA_REQ_WAIT: 2350 (void)stop_ep_timer(ep); 2351 break; 2352 case MPA_REQ_SENT: 2353 (void)stop_ep_timer(ep); 2354 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1)) 2355 connect_reply_upcall(ep, -ECONNRESET); 2356 else { 2357 /* 2358 * we just don't send notification upwards because we 2359 * want to retry with mpa_v1 without upper layers even 2360 * knowing it. 2361 * 2362 * do some housekeeping so as to re-initiate the 2363 * connection 2364 */ 2365 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 2366 mpa_rev); 2367 ep->retry_with_mpa_v1 = 1; 2368 } 2369 break; 2370 case MPA_REP_SENT: 2371 break; 2372 case MPA_REQ_RCVD: 2373 break; 2374 case MORIBUND: 2375 case CLOSING: 2376 stop_ep_timer(ep); 2377 /*FALLTHROUGH*/ 2378 case FPDU_MODE: 2379 if (ep->com.cm_id && ep->com.qp) { 2380 attrs.next_state = C4IW_QP_STATE_ERROR; 2381 ret = c4iw_modify_qp(ep->com.qp->rhp, 2382 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2383 &attrs, 1); 2384 if (ret) 2385 printk(KERN_ERR MOD 2386 "%s - qp <- error failed!\n", 2387 __func__); 2388 } 2389 peer_abort_upcall(ep); 2390 break; 2391 case ABORTING: 2392 break; 2393 case DEAD: 2394 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 2395 mutex_unlock(&ep->com.mutex); 2396 return 0; 2397 default: 2398 BUG_ON(1); 2399 break; 2400 } 2401 dst_confirm(ep->dst); 2402 if (ep->com.state != ABORTING) { 2403 __state_set(&ep->com, DEAD); 2404 /* we don't release if we want to retry with mpa_v1 */ 2405 if (!ep->retry_with_mpa_v1) 2406 release = 1; 2407 } 2408 mutex_unlock(&ep->com.mutex); 2409 2410 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 2411 if (!rpl_skb) { 2412 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 2413 __func__); 2414 release = 1; 2415 goto out; 2416 } 2417 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 2418 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 2419 INIT_TP_WR(rpl, ep->hwtid); 2420 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 2421 rpl->cmd = CPL_ABORT_NO_RST; 2422 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 2423 out: 2424 if (release) 2425 release_ep_resources(ep); 2426 else if (ep->retry_with_mpa_v1) { 2427 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 2428 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 2429 dst_release(ep->dst); 2430 cxgb4_l2t_release(ep->l2t); 2431 c4iw_reconnect(ep); 2432 } 2433 2434 return 0; 2435 } 2436 2437 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2438 { 2439 struct c4iw_ep *ep; 2440 struct c4iw_qp_attributes attrs; 2441 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2442 int release = 0; 2443 struct tid_info *t = dev->rdev.lldi.tids; 2444 unsigned int tid = GET_TID(rpl); 2445 2446 ep = lookup_tid(t, tid); 2447 2448 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2449 BUG_ON(!ep); 2450 2451 /* The cm_id may be null if we failed to connect */ 2452 mutex_lock(&ep->com.mutex); 2453 switch (ep->com.state) { 2454 case CLOSING: 2455 __state_set(&ep->com, MORIBUND); 2456 break; 2457 case MORIBUND: 2458 (void)stop_ep_timer(ep); 2459 if ((ep->com.cm_id) && (ep->com.qp)) { 2460 attrs.next_state = C4IW_QP_STATE_IDLE; 2461 c4iw_modify_qp(ep->com.qp->rhp, 2462 ep->com.qp, 2463 C4IW_QP_ATTR_NEXT_STATE, 2464 &attrs, 1); 2465 } 2466 close_complete_upcall(ep, 0); 2467 __state_set(&ep->com, DEAD); 2468 release = 1; 2469 break; 2470 case ABORTING: 2471 case DEAD: 2472 break; 2473 default: 2474 BUG_ON(1); 2475 break; 2476 } 2477 mutex_unlock(&ep->com.mutex); 2478 if (release) 2479 release_ep_resources(ep); 2480 return 0; 2481 } 2482 2483 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2484 { 2485 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2486 struct tid_info *t = dev->rdev.lldi.tids; 2487 unsigned int tid = GET_TID(rpl); 2488 struct c4iw_ep *ep; 2489 struct c4iw_qp_attributes attrs; 2490 2491 ep = lookup_tid(t, tid); 2492 BUG_ON(!ep); 2493 2494 if (ep && ep->com.qp) { 2495 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2496 ep->com.qp->wq.sq.qid); 2497 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2498 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2499 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2500 } else 2501 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2502 2503 return 0; 2504 } 2505 2506 /* 2507 * Upcall from the adapter indicating data has been transmitted. 2508 * For us its just the single MPA request or reply. We can now free 2509 * the skb holding the mpa message. 2510 */ 2511 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2512 { 2513 struct c4iw_ep *ep; 2514 struct cpl_fw4_ack *hdr = cplhdr(skb); 2515 u8 credits = hdr->credits; 2516 unsigned int tid = GET_TID(hdr); 2517 struct tid_info *t = dev->rdev.lldi.tids; 2518 2519 2520 ep = lookup_tid(t, tid); 2521 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2522 if (credits == 0) { 2523 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2524 __func__, ep, ep->hwtid, state_read(&ep->com)); 2525 return 0; 2526 } 2527 2528 dst_confirm(ep->dst); 2529 if (ep->mpa_skb) { 2530 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2531 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2532 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2533 kfree_skb(ep->mpa_skb); 2534 ep->mpa_skb = NULL; 2535 } 2536 return 0; 2537 } 2538 2539 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2540 { 2541 int err = 0; 2542 int disconnect = 0; 2543 struct c4iw_ep *ep = to_ep(cm_id); 2544 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2545 2546 mutex_lock(&ep->com.mutex); 2547 if (ep->com.state == DEAD) { 2548 mutex_unlock(&ep->com.mutex); 2549 c4iw_put_ep(&ep->com); 2550 return -ECONNRESET; 2551 } 2552 set_bit(ULP_REJECT, &ep->com.history); 2553 BUG_ON(ep->com.state != MPA_REQ_RCVD); 2554 if (mpa_rev == 0) 2555 abort_connection(ep, NULL, GFP_KERNEL); 2556 else { 2557 err = send_mpa_reject(ep, pdata, pdata_len); 2558 disconnect = 1; 2559 } 2560 mutex_unlock(&ep->com.mutex); 2561 if (disconnect) 2562 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2563 c4iw_put_ep(&ep->com); 2564 return 0; 2565 } 2566 2567 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2568 { 2569 int err; 2570 struct c4iw_qp_attributes attrs; 2571 enum c4iw_qp_attr_mask mask; 2572 struct c4iw_ep *ep = to_ep(cm_id); 2573 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2574 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2575 2576 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2577 2578 mutex_lock(&ep->com.mutex); 2579 if (ep->com.state == DEAD) { 2580 err = -ECONNRESET; 2581 goto err; 2582 } 2583 2584 BUG_ON(ep->com.state != MPA_REQ_RCVD); 2585 BUG_ON(!qp); 2586 2587 set_bit(ULP_ACCEPT, &ep->com.history); 2588 if ((conn_param->ord > c4iw_max_read_depth) || 2589 (conn_param->ird > c4iw_max_read_depth)) { 2590 abort_connection(ep, NULL, GFP_KERNEL); 2591 err = -EINVAL; 2592 goto err; 2593 } 2594 2595 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2596 if (conn_param->ord > ep->ird) { 2597 ep->ird = conn_param->ird; 2598 ep->ord = conn_param->ord; 2599 send_mpa_reject(ep, conn_param->private_data, 2600 conn_param->private_data_len); 2601 abort_connection(ep, NULL, GFP_KERNEL); 2602 err = -ENOMEM; 2603 goto err; 2604 } 2605 if (conn_param->ird > ep->ord) { 2606 if (!ep->ord) 2607 conn_param->ird = 1; 2608 else { 2609 abort_connection(ep, NULL, GFP_KERNEL); 2610 err = -ENOMEM; 2611 goto err; 2612 } 2613 } 2614 2615 } 2616 ep->ird = conn_param->ird; 2617 ep->ord = conn_param->ord; 2618 2619 if (ep->mpa_attr.version != 2) 2620 if (peer2peer && ep->ird == 0) 2621 ep->ird = 1; 2622 2623 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2624 2625 cm_id->add_ref(cm_id); 2626 ep->com.cm_id = cm_id; 2627 ep->com.qp = qp; 2628 ref_qp(ep); 2629 2630 /* bind QP to EP and move to RTS */ 2631 attrs.mpa_attr = ep->mpa_attr; 2632 attrs.max_ird = ep->ird; 2633 attrs.max_ord = ep->ord; 2634 attrs.llp_stream_handle = ep; 2635 attrs.next_state = C4IW_QP_STATE_RTS; 2636 2637 /* bind QP and TID with INIT_WR */ 2638 mask = C4IW_QP_ATTR_NEXT_STATE | 2639 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2640 C4IW_QP_ATTR_MPA_ATTR | 2641 C4IW_QP_ATTR_MAX_IRD | 2642 C4IW_QP_ATTR_MAX_ORD; 2643 2644 err = c4iw_modify_qp(ep->com.qp->rhp, 2645 ep->com.qp, mask, &attrs, 1); 2646 if (err) 2647 goto err1; 2648 err = send_mpa_reply(ep, conn_param->private_data, 2649 conn_param->private_data_len); 2650 if (err) 2651 goto err1; 2652 2653 __state_set(&ep->com, FPDU_MODE); 2654 established_upcall(ep); 2655 mutex_unlock(&ep->com.mutex); 2656 c4iw_put_ep(&ep->com); 2657 return 0; 2658 err1: 2659 ep->com.cm_id = NULL; 2660 cm_id->rem_ref(cm_id); 2661 err: 2662 mutex_unlock(&ep->com.mutex); 2663 c4iw_put_ep(&ep->com); 2664 return err; 2665 } 2666 2667 static int pick_local_ipaddrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2668 { 2669 struct in_device *ind; 2670 int found = 0; 2671 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2672 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2673 2674 ind = in_dev_get(dev->rdev.lldi.ports[0]); 2675 if (!ind) 2676 return -EADDRNOTAVAIL; 2677 for_primary_ifa(ind) { 2678 laddr->sin_addr.s_addr = ifa->ifa_address; 2679 raddr->sin_addr.s_addr = ifa->ifa_address; 2680 found = 1; 2681 break; 2682 } 2683 endfor_ifa(ind); 2684 in_dev_put(ind); 2685 return found ? 0 : -EADDRNOTAVAIL; 2686 } 2687 2688 static int get_lladdr(struct net_device *dev, struct in6_addr *addr, 2689 unsigned char banned_flags) 2690 { 2691 struct inet6_dev *idev; 2692 int err = -EADDRNOTAVAIL; 2693 2694 rcu_read_lock(); 2695 idev = __in6_dev_get(dev); 2696 if (idev != NULL) { 2697 struct inet6_ifaddr *ifp; 2698 2699 read_lock_bh(&idev->lock); 2700 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2701 if (ifp->scope == IFA_LINK && 2702 !(ifp->flags & banned_flags)) { 2703 memcpy(addr, &ifp->addr, 16); 2704 err = 0; 2705 break; 2706 } 2707 } 2708 read_unlock_bh(&idev->lock); 2709 } 2710 rcu_read_unlock(); 2711 return err; 2712 } 2713 2714 static int pick_local_ip6addrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2715 { 2716 struct in6_addr uninitialized_var(addr); 2717 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2718 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&cm_id->remote_addr; 2719 2720 if (get_lladdr(dev->rdev.lldi.ports[0], &addr, IFA_F_TENTATIVE)) { 2721 memcpy(la6->sin6_addr.s6_addr, &addr, 16); 2722 memcpy(ra6->sin6_addr.s6_addr, &addr, 16); 2723 return 0; 2724 } 2725 return -EADDRNOTAVAIL; 2726 } 2727 2728 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2729 { 2730 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2731 struct c4iw_ep *ep; 2732 int err = 0; 2733 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2734 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2735 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2736 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 2737 &cm_id->remote_addr; 2738 __u8 *ra; 2739 int iptype; 2740 2741 if ((conn_param->ord > c4iw_max_read_depth) || 2742 (conn_param->ird > c4iw_max_read_depth)) { 2743 err = -EINVAL; 2744 goto out; 2745 } 2746 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2747 if (!ep) { 2748 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2749 err = -ENOMEM; 2750 goto out; 2751 } 2752 init_timer(&ep->timer); 2753 ep->plen = conn_param->private_data_len; 2754 if (ep->plen) 2755 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2756 conn_param->private_data, ep->plen); 2757 ep->ird = conn_param->ird; 2758 ep->ord = conn_param->ord; 2759 2760 if (peer2peer && ep->ord == 0) 2761 ep->ord = 1; 2762 2763 cm_id->add_ref(cm_id); 2764 ep->com.dev = dev; 2765 ep->com.cm_id = cm_id; 2766 ep->com.qp = get_qhp(dev, conn_param->qpn); 2767 if (!ep->com.qp) { 2768 PDBG("%s qpn 0x%x not found!\n", __func__, conn_param->qpn); 2769 err = -EINVAL; 2770 goto fail2; 2771 } 2772 ref_qp(ep); 2773 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2774 ep->com.qp, cm_id); 2775 2776 /* 2777 * Allocate an active TID to initiate a TCP connection. 2778 */ 2779 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2780 if (ep->atid == -1) { 2781 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2782 err = -ENOMEM; 2783 goto fail2; 2784 } 2785 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2786 2787 if (cm_id->remote_addr.ss_family == AF_INET) { 2788 iptype = 4; 2789 ra = (__u8 *)&raddr->sin_addr; 2790 2791 /* 2792 * Handle loopback requests to INADDR_ANY. 2793 */ 2794 if ((__force int)raddr->sin_addr.s_addr == INADDR_ANY) { 2795 err = pick_local_ipaddrs(dev, cm_id); 2796 if (err) 2797 goto fail2; 2798 } 2799 2800 /* find a route */ 2801 PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n", 2802 __func__, &laddr->sin_addr, ntohs(laddr->sin_port), 2803 ra, ntohs(raddr->sin_port)); 2804 ep->dst = find_route(dev, laddr->sin_addr.s_addr, 2805 raddr->sin_addr.s_addr, laddr->sin_port, 2806 raddr->sin_port, 0); 2807 } else { 2808 iptype = 6; 2809 ra = (__u8 *)&raddr6->sin6_addr; 2810 2811 /* 2812 * Handle loopback requests to INADDR_ANY. 2813 */ 2814 if (ipv6_addr_type(&raddr6->sin6_addr) == IPV6_ADDR_ANY) { 2815 err = pick_local_ip6addrs(dev, cm_id); 2816 if (err) 2817 goto fail2; 2818 } 2819 2820 /* find a route */ 2821 PDBG("%s saddr %pI6 sport 0x%x raddr %pI6 rport 0x%x\n", 2822 __func__, laddr6->sin6_addr.s6_addr, 2823 ntohs(laddr6->sin6_port), 2824 raddr6->sin6_addr.s6_addr, ntohs(raddr6->sin6_port)); 2825 ep->dst = find_route6(dev, laddr6->sin6_addr.s6_addr, 2826 raddr6->sin6_addr.s6_addr, 2827 laddr6->sin6_port, raddr6->sin6_port, 0, 2828 raddr6->sin6_scope_id); 2829 } 2830 if (!ep->dst) { 2831 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2832 err = -EHOSTUNREACH; 2833 goto fail3; 2834 } 2835 2836 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, true); 2837 if (err) { 2838 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2839 goto fail4; 2840 } 2841 2842 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2843 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2844 ep->l2t->idx); 2845 2846 state_set(&ep->com, CONNECTING); 2847 ep->tos = 0; 2848 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2849 sizeof(ep->com.local_addr)); 2850 memcpy(&ep->com.remote_addr, &cm_id->remote_addr, 2851 sizeof(ep->com.remote_addr)); 2852 2853 /* send connect request to rnic */ 2854 err = send_connect(ep); 2855 if (!err) 2856 goto out; 2857 2858 cxgb4_l2t_release(ep->l2t); 2859 fail4: 2860 dst_release(ep->dst); 2861 fail3: 2862 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 2863 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2864 fail2: 2865 cm_id->rem_ref(cm_id); 2866 c4iw_put_ep(&ep->com); 2867 out: 2868 return err; 2869 } 2870 2871 static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2872 { 2873 int err; 2874 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ep->com.local_addr; 2875 2876 c4iw_init_wr_wait(&ep->com.wr_wait); 2877 err = cxgb4_create_server6(ep->com.dev->rdev.lldi.ports[0], 2878 ep->stid, &sin6->sin6_addr, 2879 sin6->sin6_port, 2880 ep->com.dev->rdev.lldi.rxq_ids[0]); 2881 if (!err) 2882 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2883 &ep->com.wr_wait, 2884 0, 0, __func__); 2885 if (err) 2886 pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n", 2887 err, ep->stid, 2888 sin6->sin6_addr.s6_addr, ntohs(sin6->sin6_port)); 2889 return err; 2890 } 2891 2892 static int create_server4(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2893 { 2894 int err; 2895 struct sockaddr_in *sin = (struct sockaddr_in *)&ep->com.local_addr; 2896 2897 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2898 do { 2899 err = cxgb4_create_server_filter( 2900 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2901 sin->sin_addr.s_addr, sin->sin_port, 0, 2902 ep->com.dev->rdev.lldi.rxq_ids[0], 0, 0); 2903 if (err == -EBUSY) { 2904 set_current_state(TASK_UNINTERRUPTIBLE); 2905 schedule_timeout(usecs_to_jiffies(100)); 2906 } 2907 } while (err == -EBUSY); 2908 } else { 2909 c4iw_init_wr_wait(&ep->com.wr_wait); 2910 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], 2911 ep->stid, sin->sin_addr.s_addr, sin->sin_port, 2912 0, ep->com.dev->rdev.lldi.rxq_ids[0]); 2913 if (!err) 2914 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2915 &ep->com.wr_wait, 2916 0, 0, __func__); 2917 } 2918 if (err) 2919 pr_err("cxgb4_create_server/filter failed err %d stid %d laddr %pI4 lport %d\n" 2920 , err, ep->stid, 2921 &sin->sin_addr, ntohs(sin->sin_port)); 2922 return err; 2923 } 2924 2925 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2926 { 2927 int err = 0; 2928 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2929 struct c4iw_listen_ep *ep; 2930 2931 might_sleep(); 2932 2933 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2934 if (!ep) { 2935 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2936 err = -ENOMEM; 2937 goto fail1; 2938 } 2939 PDBG("%s ep %p\n", __func__, ep); 2940 cm_id->add_ref(cm_id); 2941 ep->com.cm_id = cm_id; 2942 ep->com.dev = dev; 2943 ep->backlog = backlog; 2944 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2945 sizeof(ep->com.local_addr)); 2946 2947 /* 2948 * Allocate a server TID. 2949 */ 2950 if (dev->rdev.lldi.enable_fw_ofld_conn && 2951 ep->com.local_addr.ss_family == AF_INET) 2952 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, 2953 cm_id->local_addr.ss_family, ep); 2954 else 2955 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, 2956 cm_id->local_addr.ss_family, ep); 2957 2958 if (ep->stid == -1) { 2959 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2960 err = -ENOMEM; 2961 goto fail2; 2962 } 2963 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2964 state_set(&ep->com, LISTEN); 2965 if (ep->com.local_addr.ss_family == AF_INET) 2966 err = create_server4(dev, ep); 2967 else 2968 err = create_server6(dev, ep); 2969 if (!err) { 2970 cm_id->provider_data = ep; 2971 goto out; 2972 } 2973 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 2974 ep->com.local_addr.ss_family); 2975 fail2: 2976 cm_id->rem_ref(cm_id); 2977 c4iw_put_ep(&ep->com); 2978 fail1: 2979 out: 2980 return err; 2981 } 2982 2983 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2984 { 2985 int err; 2986 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2987 2988 PDBG("%s ep %p\n", __func__, ep); 2989 2990 might_sleep(); 2991 state_set(&ep->com, DEAD); 2992 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn && 2993 ep->com.local_addr.ss_family == AF_INET) { 2994 err = cxgb4_remove_server_filter( 2995 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2996 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2997 } else { 2998 c4iw_init_wr_wait(&ep->com.wr_wait); 2999 err = cxgb4_remove_server( 3000 ep->com.dev->rdev.lldi.ports[0], ep->stid, 3001 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 3002 if (err) 3003 goto done; 3004 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 3005 0, 0, __func__); 3006 } 3007 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 3008 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 3009 ep->com.local_addr.ss_family); 3010 done: 3011 cm_id->rem_ref(cm_id); 3012 c4iw_put_ep(&ep->com); 3013 return err; 3014 } 3015 3016 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 3017 { 3018 int ret = 0; 3019 int close = 0; 3020 int fatal = 0; 3021 struct c4iw_rdev *rdev; 3022 3023 mutex_lock(&ep->com.mutex); 3024 3025 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 3026 states[ep->com.state], abrupt); 3027 3028 rdev = &ep->com.dev->rdev; 3029 if (c4iw_fatal_error(rdev)) { 3030 fatal = 1; 3031 close_complete_upcall(ep, -EIO); 3032 ep->com.state = DEAD; 3033 } 3034 switch (ep->com.state) { 3035 case MPA_REQ_WAIT: 3036 case MPA_REQ_SENT: 3037 case MPA_REQ_RCVD: 3038 case MPA_REP_SENT: 3039 case FPDU_MODE: 3040 close = 1; 3041 if (abrupt) 3042 ep->com.state = ABORTING; 3043 else { 3044 ep->com.state = CLOSING; 3045 start_ep_timer(ep); 3046 } 3047 set_bit(CLOSE_SENT, &ep->com.flags); 3048 break; 3049 case CLOSING: 3050 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 3051 close = 1; 3052 if (abrupt) { 3053 (void)stop_ep_timer(ep); 3054 ep->com.state = ABORTING; 3055 } else 3056 ep->com.state = MORIBUND; 3057 } 3058 break; 3059 case MORIBUND: 3060 case ABORTING: 3061 case DEAD: 3062 PDBG("%s ignoring disconnect ep %p state %u\n", 3063 __func__, ep, ep->com.state); 3064 break; 3065 default: 3066 BUG(); 3067 break; 3068 } 3069 3070 if (close) { 3071 if (abrupt) { 3072 set_bit(EP_DISC_ABORT, &ep->com.history); 3073 close_complete_upcall(ep, -ECONNRESET); 3074 ret = send_abort(ep, NULL, gfp); 3075 } else { 3076 set_bit(EP_DISC_CLOSE, &ep->com.history); 3077 ret = send_halfclose(ep, gfp); 3078 } 3079 if (ret) 3080 fatal = 1; 3081 } 3082 mutex_unlock(&ep->com.mutex); 3083 if (fatal) 3084 release_ep_resources(ep); 3085 return ret; 3086 } 3087 3088 static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3089 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3090 { 3091 struct c4iw_ep *ep; 3092 int atid = be32_to_cpu(req->tid); 3093 3094 ep = (struct c4iw_ep *)lookup_atid(dev->rdev.lldi.tids, 3095 (__force u32) req->tid); 3096 if (!ep) 3097 return; 3098 3099 switch (req->retval) { 3100 case FW_ENOMEM: 3101 set_bit(ACT_RETRY_NOMEM, &ep->com.history); 3102 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3103 send_fw_act_open_req(ep, atid); 3104 return; 3105 } 3106 case FW_EADDRINUSE: 3107 set_bit(ACT_RETRY_INUSE, &ep->com.history); 3108 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3109 send_fw_act_open_req(ep, atid); 3110 return; 3111 } 3112 break; 3113 default: 3114 pr_info("%s unexpected ofld conn wr retval %d\n", 3115 __func__, req->retval); 3116 break; 3117 } 3118 pr_err("active ofld_connect_wr failure %d atid %d\n", 3119 req->retval, atid); 3120 mutex_lock(&dev->rdev.stats.lock); 3121 dev->rdev.stats.act_ofld_conn_fails++; 3122 mutex_unlock(&dev->rdev.stats.lock); 3123 connect_reply_upcall(ep, status2errno(req->retval)); 3124 state_set(&ep->com, DEAD); 3125 remove_handle(dev, &dev->atid_idr, atid); 3126 cxgb4_free_atid(dev->rdev.lldi.tids, atid); 3127 dst_release(ep->dst); 3128 cxgb4_l2t_release(ep->l2t); 3129 c4iw_put_ep(&ep->com); 3130 } 3131 3132 static void passive_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3133 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3134 { 3135 struct sk_buff *rpl_skb; 3136 struct cpl_pass_accept_req *cpl; 3137 int ret; 3138 3139 rpl_skb = (struct sk_buff *)(unsigned long)req->cookie; 3140 BUG_ON(!rpl_skb); 3141 if (req->retval) { 3142 PDBG("%s passive open failure %d\n", __func__, req->retval); 3143 mutex_lock(&dev->rdev.stats.lock); 3144 dev->rdev.stats.pas_ofld_conn_fails++; 3145 mutex_unlock(&dev->rdev.stats.lock); 3146 kfree_skb(rpl_skb); 3147 } else { 3148 cpl = (struct cpl_pass_accept_req *)cplhdr(rpl_skb); 3149 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 3150 (__force u32) htonl( 3151 (__force u32) req->tid))); 3152 ret = pass_accept_req(dev, rpl_skb); 3153 if (!ret) 3154 kfree_skb(rpl_skb); 3155 } 3156 return; 3157 } 3158 3159 static int deferred_fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3160 { 3161 struct cpl_fw6_msg *rpl = cplhdr(skb); 3162 struct cpl_fw6_msg_ofld_connection_wr_rpl *req; 3163 3164 switch (rpl->type) { 3165 case FW6_TYPE_CQE: 3166 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 3167 break; 3168 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3169 req = (struct cpl_fw6_msg_ofld_connection_wr_rpl *)rpl->data; 3170 switch (req->t_state) { 3171 case TCP_SYN_SENT: 3172 active_ofld_conn_reply(dev, skb, req); 3173 break; 3174 case TCP_SYN_RECV: 3175 passive_ofld_conn_reply(dev, skb, req); 3176 break; 3177 default: 3178 pr_err("%s unexpected ofld conn wr state %d\n", 3179 __func__, req->t_state); 3180 break; 3181 } 3182 break; 3183 } 3184 return 0; 3185 } 3186 3187 static void build_cpl_pass_accept_req(struct sk_buff *skb, int stid , u8 tos) 3188 { 3189 u32 l2info; 3190 u16 vlantag, len, hdr_len, eth_hdr_len; 3191 u8 intf; 3192 struct cpl_rx_pkt *cpl = cplhdr(skb); 3193 struct cpl_pass_accept_req *req; 3194 struct tcp_options_received tmp_opt; 3195 struct c4iw_dev *dev; 3196 3197 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3198 /* Store values from cpl_rx_pkt in temporary location. */ 3199 vlantag = (__force u16) cpl->vlan; 3200 len = (__force u16) cpl->len; 3201 l2info = (__force u32) cpl->l2info; 3202 hdr_len = (__force u16) cpl->hdr_len; 3203 intf = cpl->iff; 3204 3205 __skb_pull(skb, sizeof(*req) + sizeof(struct rss_header)); 3206 3207 /* 3208 * We need to parse the TCP options from SYN packet. 3209 * to generate cpl_pass_accept_req. 3210 */ 3211 memset(&tmp_opt, 0, sizeof(tmp_opt)); 3212 tcp_clear_options(&tmp_opt); 3213 tcp_parse_options(skb, &tmp_opt, 0, NULL); 3214 3215 req = (struct cpl_pass_accept_req *)__skb_push(skb, sizeof(*req)); 3216 memset(req, 0, sizeof(*req)); 3217 req->l2info = cpu_to_be16(V_SYN_INTF(intf) | 3218 V_SYN_MAC_IDX(G_RX_MACIDX( 3219 (__force int) htonl(l2info))) | 3220 F_SYN_XACT_MATCH); 3221 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3222 G_RX_ETHHDR_LEN((__force int) htonl(l2info)) : 3223 G_RX_T5_ETHHDR_LEN((__force int) htonl(l2info)); 3224 req->hdr_len = cpu_to_be32(V_SYN_RX_CHAN(G_RX_CHAN( 3225 (__force int) htonl(l2info))) | 3226 V_TCP_HDR_LEN(G_RX_TCPHDR_LEN( 3227 (__force int) htons(hdr_len))) | 3228 V_IP_HDR_LEN(G_RX_IPHDR_LEN( 3229 (__force int) htons(hdr_len))) | 3230 V_ETH_HDR_LEN(G_RX_ETHHDR_LEN(eth_hdr_len))); 3231 req->vlan = (__force __be16) vlantag; 3232 req->len = (__force __be16) len; 3233 req->tos_stid = cpu_to_be32(PASS_OPEN_TID(stid) | 3234 PASS_OPEN_TOS(tos)); 3235 req->tcpopt.mss = htons(tmp_opt.mss_clamp); 3236 if (tmp_opt.wscale_ok) 3237 req->tcpopt.wsf = tmp_opt.snd_wscale; 3238 req->tcpopt.tstamp = tmp_opt.saw_tstamp; 3239 if (tmp_opt.sack_ok) 3240 req->tcpopt.sack = 1; 3241 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 0)); 3242 return; 3243 } 3244 3245 static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, 3246 __be32 laddr, __be16 lport, 3247 __be32 raddr, __be16 rport, 3248 u32 rcv_isn, u32 filter, u16 window, 3249 u32 rss_qid, u8 port_id) 3250 { 3251 struct sk_buff *req_skb; 3252 struct fw_ofld_connection_wr *req; 3253 struct cpl_pass_accept_req *cpl = cplhdr(skb); 3254 int ret; 3255 3256 req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); 3257 req = (struct fw_ofld_connection_wr *)__skb_put(req_skb, sizeof(*req)); 3258 memset(req, 0, sizeof(*req)); 3259 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL(1)); 3260 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 3261 req->le.version_cpl = htonl(F_FW_OFLD_CONNECTION_WR_CPL); 3262 req->le.filter = (__force __be32) filter; 3263 req->le.lport = lport; 3264 req->le.pport = rport; 3265 req->le.u.ipv4.lip = laddr; 3266 req->le.u.ipv4.pip = raddr; 3267 req->tcb.rcv_nxt = htonl(rcv_isn + 1); 3268 req->tcb.rcv_adv = htons(window); 3269 req->tcb.t_state_to_astid = 3270 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_RECV) | 3271 V_FW_OFLD_CONNECTION_WR_RCV_SCALE(cpl->tcpopt.wsf) | 3272 V_FW_OFLD_CONNECTION_WR_ASTID( 3273 GET_PASS_OPEN_TID(ntohl(cpl->tos_stid)))); 3274 3275 /* 3276 * We store the qid in opt2 which will be used by the firmware 3277 * to send us the wr response. 3278 */ 3279 req->tcb.opt2 = htonl(V_RSS_QUEUE(rss_qid)); 3280 3281 /* 3282 * We initialize the MSS index in TCB to 0xF. 3283 * So that when driver sends cpl_pass_accept_rpl 3284 * TCB picks up the correct value. If this was 0 3285 * TP will ignore any value > 0 for MSS index. 3286 */ 3287 req->tcb.opt0 = cpu_to_be64(V_MSS_IDX(0xF)); 3288 req->cookie = (unsigned long)skb; 3289 3290 set_wr_txq(req_skb, CPL_PRIORITY_CONTROL, port_id); 3291 ret = cxgb4_ofld_send(dev->rdev.lldi.ports[0], req_skb); 3292 if (ret < 0) { 3293 pr_err("%s - cxgb4_ofld_send error %d - dropping\n", __func__, 3294 ret); 3295 kfree_skb(skb); 3296 kfree_skb(req_skb); 3297 } 3298 } 3299 3300 /* 3301 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt 3302 * messages when a filter is being used instead of server to 3303 * redirect a syn packet. When packets hit filter they are redirected 3304 * to the offload queue and driver tries to establish the connection 3305 * using firmware work request. 3306 */ 3307 static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) 3308 { 3309 int stid; 3310 unsigned int filter; 3311 struct ethhdr *eh = NULL; 3312 struct vlan_ethhdr *vlan_eh = NULL; 3313 struct iphdr *iph; 3314 struct tcphdr *tcph; 3315 struct rss_header *rss = (void *)skb->data; 3316 struct cpl_rx_pkt *cpl = (void *)skb->data; 3317 struct cpl_pass_accept_req *req = (void *)(rss + 1); 3318 struct l2t_entry *e; 3319 struct dst_entry *dst; 3320 struct c4iw_ep *lep; 3321 u16 window; 3322 struct port_info *pi; 3323 struct net_device *pdev; 3324 u16 rss_qid, eth_hdr_len; 3325 int step; 3326 u32 tx_chan; 3327 struct neighbour *neigh; 3328 3329 /* Drop all non-SYN packets */ 3330 if (!(cpl->l2info & cpu_to_be32(F_RXF_SYN))) 3331 goto reject; 3332 3333 /* 3334 * Drop all packets which did not hit the filter. 3335 * Unlikely to happen. 3336 */ 3337 if (!(rss->filter_hit && rss->filter_tid)) 3338 goto reject; 3339 3340 /* 3341 * Calculate the server tid from filter hit index from cpl_rx_pkt. 3342 */ 3343 stid = (__force int) cpu_to_be32((__force u32) rss->hash_val); 3344 3345 lep = (struct c4iw_ep *)lookup_stid(dev->rdev.lldi.tids, stid); 3346 if (!lep) { 3347 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 3348 goto reject; 3349 } 3350 3351 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3352 G_RX_ETHHDR_LEN(htonl(cpl->l2info)) : 3353 G_RX_T5_ETHHDR_LEN(htonl(cpl->l2info)); 3354 if (eth_hdr_len == ETH_HLEN) { 3355 eh = (struct ethhdr *)(req + 1); 3356 iph = (struct iphdr *)(eh + 1); 3357 } else { 3358 vlan_eh = (struct vlan_ethhdr *)(req + 1); 3359 iph = (struct iphdr *)(vlan_eh + 1); 3360 skb->vlan_tci = ntohs(cpl->vlan); 3361 } 3362 3363 if (iph->version != 0x4) 3364 goto reject; 3365 3366 tcph = (struct tcphdr *)(iph + 1); 3367 skb_set_network_header(skb, (void *)iph - (void *)rss); 3368 skb_set_transport_header(skb, (void *)tcph - (void *)rss); 3369 skb_get(skb); 3370 3371 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__, 3372 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3373 ntohs(tcph->source), iph->tos); 3374 3375 dst = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3376 iph->tos); 3377 if (!dst) { 3378 pr_err("%s - failed to find dst entry!\n", 3379 __func__); 3380 goto reject; 3381 } 3382 neigh = dst_neigh_lookup_skb(dst, skb); 3383 3384 if (!neigh) { 3385 pr_err("%s - failed to allocate neigh!\n", 3386 __func__); 3387 goto free_dst; 3388 } 3389 3390 if (neigh->dev->flags & IFF_LOOPBACK) { 3391 pdev = ip_dev_find(&init_net, iph->daddr); 3392 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3393 pdev, 0); 3394 pi = (struct port_info *)netdev_priv(pdev); 3395 tx_chan = cxgb4_port_chan(pdev); 3396 dev_put(pdev); 3397 } else { 3398 pdev = get_real_dev(neigh->dev); 3399 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3400 pdev, 0); 3401 pi = (struct port_info *)netdev_priv(pdev); 3402 tx_chan = cxgb4_port_chan(pdev); 3403 } 3404 neigh_release(neigh); 3405 if (!e) { 3406 pr_err("%s - failed to allocate l2t entry!\n", 3407 __func__); 3408 goto free_dst; 3409 } 3410 3411 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; 3412 rss_qid = dev->rdev.lldi.rxq_ids[pi->port_id * step]; 3413 window = (__force u16) htons((__force u16)tcph->window); 3414 3415 /* Calcuate filter portion for LE region. */ 3416 filter = (__force unsigned int) cpu_to_be32(cxgb4_select_ntuple( 3417 dev->rdev.lldi.ports[0], 3418 e)); 3419 3420 /* 3421 * Synthesize the cpl_pass_accept_req. We have everything except the 3422 * TID. Once firmware sends a reply with TID we update the TID field 3423 * in cpl and pass it through the regular cpl_pass_accept_req path. 3424 */ 3425 build_cpl_pass_accept_req(skb, stid, iph->tos); 3426 send_fw_pass_open_req(dev, skb, iph->daddr, tcph->dest, iph->saddr, 3427 tcph->source, ntohl(tcph->seq), filter, window, 3428 rss_qid, pi->port_id); 3429 cxgb4_l2t_release(e); 3430 free_dst: 3431 dst_release(dst); 3432 reject: 3433 return 0; 3434 } 3435 3436 /* 3437 * These are the real handlers that are called from a 3438 * work queue. 3439 */ 3440 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 3441 [CPL_ACT_ESTABLISH] = act_establish, 3442 [CPL_ACT_OPEN_RPL] = act_open_rpl, 3443 [CPL_RX_DATA] = rx_data, 3444 [CPL_ABORT_RPL_RSS] = abort_rpl, 3445 [CPL_ABORT_RPL] = abort_rpl, 3446 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 3447 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 3448 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 3449 [CPL_PASS_ESTABLISH] = pass_establish, 3450 [CPL_PEER_CLOSE] = peer_close, 3451 [CPL_ABORT_REQ_RSS] = peer_abort, 3452 [CPL_CLOSE_CON_RPL] = close_con_rpl, 3453 [CPL_RDMA_TERMINATE] = terminate, 3454 [CPL_FW4_ACK] = fw4_ack, 3455 [CPL_FW6_MSG] = deferred_fw6_msg, 3456 [CPL_RX_PKT] = rx_pkt 3457 }; 3458 3459 static void process_timeout(struct c4iw_ep *ep) 3460 { 3461 struct c4iw_qp_attributes attrs; 3462 int abort = 1; 3463 3464 mutex_lock(&ep->com.mutex); 3465 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 3466 ep->com.state); 3467 set_bit(TIMEDOUT, &ep->com.history); 3468 switch (ep->com.state) { 3469 case MPA_REQ_SENT: 3470 __state_set(&ep->com, ABORTING); 3471 connect_reply_upcall(ep, -ETIMEDOUT); 3472 break; 3473 case MPA_REQ_WAIT: 3474 __state_set(&ep->com, ABORTING); 3475 break; 3476 case CLOSING: 3477 case MORIBUND: 3478 if (ep->com.cm_id && ep->com.qp) { 3479 attrs.next_state = C4IW_QP_STATE_ERROR; 3480 c4iw_modify_qp(ep->com.qp->rhp, 3481 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 3482 &attrs, 1); 3483 } 3484 __state_set(&ep->com, ABORTING); 3485 close_complete_upcall(ep, -ETIMEDOUT); 3486 break; 3487 case ABORTING: 3488 case DEAD: 3489 3490 /* 3491 * These states are expected if the ep timed out at the same 3492 * time as another thread was calling stop_ep_timer(). 3493 * So we silently do nothing for these states. 3494 */ 3495 abort = 0; 3496 break; 3497 default: 3498 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3499 __func__, ep, ep->hwtid, ep->com.state); 3500 abort = 0; 3501 } 3502 if (abort) 3503 abort_connection(ep, NULL, GFP_KERNEL); 3504 mutex_unlock(&ep->com.mutex); 3505 c4iw_put_ep(&ep->com); 3506 } 3507 3508 static void process_timedout_eps(void) 3509 { 3510 struct c4iw_ep *ep; 3511 3512 spin_lock_irq(&timeout_lock); 3513 while (!list_empty(&timeout_list)) { 3514 struct list_head *tmp; 3515 3516 tmp = timeout_list.next; 3517 list_del(tmp); 3518 tmp->next = NULL; 3519 tmp->prev = NULL; 3520 spin_unlock_irq(&timeout_lock); 3521 ep = list_entry(tmp, struct c4iw_ep, entry); 3522 process_timeout(ep); 3523 spin_lock_irq(&timeout_lock); 3524 } 3525 spin_unlock_irq(&timeout_lock); 3526 } 3527 3528 static void process_work(struct work_struct *work) 3529 { 3530 struct sk_buff *skb = NULL; 3531 struct c4iw_dev *dev; 3532 struct cpl_act_establish *rpl; 3533 unsigned int opcode; 3534 int ret; 3535 3536 process_timedout_eps(); 3537 while ((skb = skb_dequeue(&rxq))) { 3538 rpl = cplhdr(skb); 3539 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3540 opcode = rpl->ot.opcode; 3541 3542 BUG_ON(!work_handlers[opcode]); 3543 ret = work_handlers[opcode](dev, skb); 3544 if (!ret) 3545 kfree_skb(skb); 3546 process_timedout_eps(); 3547 } 3548 } 3549 3550 static DECLARE_WORK(skb_work, process_work); 3551 3552 static void ep_timeout(unsigned long arg) 3553 { 3554 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 3555 int kickit = 0; 3556 3557 spin_lock(&timeout_lock); 3558 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 3559 /* 3560 * Only insert if it is not already on the list. 3561 */ 3562 if (!ep->entry.next) { 3563 list_add_tail(&ep->entry, &timeout_list); 3564 kickit = 1; 3565 } 3566 } 3567 spin_unlock(&timeout_lock); 3568 if (kickit) 3569 queue_work(workq, &skb_work); 3570 } 3571 3572 /* 3573 * All the CM events are handled on a work queue to have a safe context. 3574 */ 3575 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 3576 { 3577 3578 /* 3579 * Save dev in the skb->cb area. 3580 */ 3581 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 3582 3583 /* 3584 * Queue the skb and schedule the worker thread. 3585 */ 3586 skb_queue_tail(&rxq, skb); 3587 queue_work(workq, &skb_work); 3588 return 0; 3589 } 3590 3591 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 3592 { 3593 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 3594 3595 if (rpl->status != CPL_ERR_NONE) { 3596 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 3597 "for tid %u\n", rpl->status, GET_TID(rpl)); 3598 } 3599 kfree_skb(skb); 3600 return 0; 3601 } 3602 3603 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3604 { 3605 struct cpl_fw6_msg *rpl = cplhdr(skb); 3606 struct c4iw_wr_wait *wr_waitp; 3607 int ret; 3608 3609 PDBG("%s type %u\n", __func__, rpl->type); 3610 3611 switch (rpl->type) { 3612 case FW6_TYPE_WR_RPL: 3613 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 3614 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 3615 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 3616 if (wr_waitp) 3617 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 3618 kfree_skb(skb); 3619 break; 3620 case FW6_TYPE_CQE: 3621 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3622 sched(dev, skb); 3623 break; 3624 default: 3625 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 3626 rpl->type); 3627 kfree_skb(skb); 3628 break; 3629 } 3630 return 0; 3631 } 3632 3633 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 3634 { 3635 struct cpl_abort_req_rss *req = cplhdr(skb); 3636 struct c4iw_ep *ep; 3637 struct tid_info *t = dev->rdev.lldi.tids; 3638 unsigned int tid = GET_TID(req); 3639 3640 ep = lookup_tid(t, tid); 3641 if (!ep) { 3642 printk(KERN_WARNING MOD 3643 "Abort on non-existent endpoint, tid %d\n", tid); 3644 kfree_skb(skb); 3645 return 0; 3646 } 3647 if (is_neg_adv(req->status)) { 3648 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 3649 ep->hwtid); 3650 kfree_skb(skb); 3651 return 0; 3652 } 3653 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 3654 ep->com.state); 3655 3656 /* 3657 * Wake up any threads in rdma_init() or rdma_fini(). 3658 * However, if we are on MPAv2 and want to retry with MPAv1 3659 * then, don't wake up yet. 3660 */ 3661 if (mpa_rev == 2 && !ep->tried_with_mpa_v1) { 3662 if (ep->com.state != MPA_REQ_SENT) 3663 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3664 } else 3665 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3666 sched(dev, skb); 3667 return 0; 3668 } 3669 3670 /* 3671 * Most upcalls from the T4 Core go to sched() to 3672 * schedule the processing on a work queue. 3673 */ 3674 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 3675 [CPL_ACT_ESTABLISH] = sched, 3676 [CPL_ACT_OPEN_RPL] = sched, 3677 [CPL_RX_DATA] = sched, 3678 [CPL_ABORT_RPL_RSS] = sched, 3679 [CPL_ABORT_RPL] = sched, 3680 [CPL_PASS_OPEN_RPL] = sched, 3681 [CPL_CLOSE_LISTSRV_RPL] = sched, 3682 [CPL_PASS_ACCEPT_REQ] = sched, 3683 [CPL_PASS_ESTABLISH] = sched, 3684 [CPL_PEER_CLOSE] = sched, 3685 [CPL_CLOSE_CON_RPL] = sched, 3686 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 3687 [CPL_RDMA_TERMINATE] = sched, 3688 [CPL_FW4_ACK] = sched, 3689 [CPL_SET_TCB_RPL] = set_tcb_rpl, 3690 [CPL_FW6_MSG] = fw6_msg, 3691 [CPL_RX_PKT] = sched 3692 }; 3693 3694 int __init c4iw_cm_init(void) 3695 { 3696 spin_lock_init(&timeout_lock); 3697 skb_queue_head_init(&rxq); 3698 3699 workq = create_singlethread_workqueue("iw_cxgb4"); 3700 if (!workq) 3701 return -ENOMEM; 3702 3703 return 0; 3704 } 3705 3706 void __exit c4iw_cm_term(void) 3707 { 3708 WARN_ON(!list_empty(&timeout_list)); 3709 flush_workqueue(workq); 3710 destroy_workqueue(workq); 3711 } 3712