1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 #include <linux/if_vlan.h> 42 43 #include <net/neighbour.h> 44 #include <net/netevent.h> 45 #include <net/route.h> 46 #include <net/tcp.h> 47 #include <net/ip6_route.h> 48 #include <net/addrconf.h> 49 50 #include "iw_cxgb4.h" 51 52 static char *states[] = { 53 "idle", 54 "listen", 55 "connecting", 56 "mpa_wait_req", 57 "mpa_req_sent", 58 "mpa_req_rcvd", 59 "mpa_rep_sent", 60 "fpdu_mode", 61 "aborting", 62 "closing", 63 "moribund", 64 "dead", 65 NULL, 66 }; 67 68 static int nocong; 69 module_param(nocong, int, 0644); 70 MODULE_PARM_DESC(nocong, "Turn of congestion control (default=0)"); 71 72 static int enable_ecn; 73 module_param(enable_ecn, int, 0644); 74 MODULE_PARM_DESC(enable_ecn, "Enable ECN (default=0/disabled)"); 75 76 static int dack_mode = 1; 77 module_param(dack_mode, int, 0644); 78 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 79 80 int c4iw_max_read_depth = 8; 81 module_param(c4iw_max_read_depth, int, 0644); 82 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 83 84 static int enable_tcp_timestamps; 85 module_param(enable_tcp_timestamps, int, 0644); 86 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 87 88 static int enable_tcp_sack; 89 module_param(enable_tcp_sack, int, 0644); 90 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 91 92 static int enable_tcp_window_scaling = 1; 93 module_param(enable_tcp_window_scaling, int, 0644); 94 MODULE_PARM_DESC(enable_tcp_window_scaling, 95 "Enable tcp window scaling (default=1)"); 96 97 int c4iw_debug; 98 module_param(c4iw_debug, int, 0644); 99 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 100 101 static int peer2peer = 1; 102 module_param(peer2peer, int, 0644); 103 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=1)"); 104 105 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 106 module_param(p2p_type, int, 0644); 107 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 108 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 109 110 static int ep_timeout_secs = 60; 111 module_param(ep_timeout_secs, int, 0644); 112 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 113 "in seconds (default=60)"); 114 115 static int mpa_rev = 1; 116 module_param(mpa_rev, int, 0644); 117 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 118 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 119 " compliant (default=1)"); 120 121 static int markers_enabled; 122 module_param(markers_enabled, int, 0644); 123 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 124 125 static int crc_enabled = 1; 126 module_param(crc_enabled, int, 0644); 127 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 128 129 static int rcv_win = 256 * 1024; 130 module_param(rcv_win, int, 0644); 131 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 132 133 static int snd_win = 128 * 1024; 134 module_param(snd_win, int, 0644); 135 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 136 137 static struct workqueue_struct *workq; 138 139 static struct sk_buff_head rxq; 140 141 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 142 static void ep_timeout(unsigned long arg); 143 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 144 145 static LIST_HEAD(timeout_list); 146 static spinlock_t timeout_lock; 147 148 static void deref_qp(struct c4iw_ep *ep) 149 { 150 c4iw_qp_rem_ref(&ep->com.qp->ibqp); 151 clear_bit(QP_REFERENCED, &ep->com.flags); 152 } 153 154 static void ref_qp(struct c4iw_ep *ep) 155 { 156 set_bit(QP_REFERENCED, &ep->com.flags); 157 c4iw_qp_add_ref(&ep->com.qp->ibqp); 158 } 159 160 static void start_ep_timer(struct c4iw_ep *ep) 161 { 162 PDBG("%s ep %p\n", __func__, ep); 163 if (timer_pending(&ep->timer)) { 164 pr_err("%s timer already started! ep %p\n", 165 __func__, ep); 166 return; 167 } 168 clear_bit(TIMEOUT, &ep->com.flags); 169 c4iw_get_ep(&ep->com); 170 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 171 ep->timer.data = (unsigned long)ep; 172 ep->timer.function = ep_timeout; 173 add_timer(&ep->timer); 174 } 175 176 static void stop_ep_timer(struct c4iw_ep *ep) 177 { 178 PDBG("%s ep %p stopping\n", __func__, ep); 179 del_timer_sync(&ep->timer); 180 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) 181 c4iw_put_ep(&ep->com); 182 } 183 184 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 185 struct l2t_entry *l2e) 186 { 187 int error = 0; 188 189 if (c4iw_fatal_error(rdev)) { 190 kfree_skb(skb); 191 PDBG("%s - device in error state - dropping\n", __func__); 192 return -EIO; 193 } 194 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 195 if (error < 0) 196 kfree_skb(skb); 197 return error < 0 ? error : 0; 198 } 199 200 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 201 { 202 int error = 0; 203 204 if (c4iw_fatal_error(rdev)) { 205 kfree_skb(skb); 206 PDBG("%s - device in error state - dropping\n", __func__); 207 return -EIO; 208 } 209 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 210 if (error < 0) 211 kfree_skb(skb); 212 return error < 0 ? error : 0; 213 } 214 215 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 216 { 217 struct cpl_tid_release *req; 218 219 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 220 if (!skb) 221 return; 222 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 223 INIT_TP_WR(req, hwtid); 224 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 225 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 226 c4iw_ofld_send(rdev, skb); 227 return; 228 } 229 230 static void set_emss(struct c4iw_ep *ep, u16 opt) 231 { 232 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 233 ep->mss = ep->emss; 234 if (GET_TCPOPT_TSTAMP(opt)) 235 ep->emss -= 12; 236 if (ep->emss < 128) 237 ep->emss = 128; 238 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 239 ep->mss, ep->emss); 240 } 241 242 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 243 { 244 enum c4iw_ep_state state; 245 246 mutex_lock(&epc->mutex); 247 state = epc->state; 248 mutex_unlock(&epc->mutex); 249 return state; 250 } 251 252 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 253 { 254 epc->state = new; 255 } 256 257 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 258 { 259 mutex_lock(&epc->mutex); 260 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 261 __state_set(epc, new); 262 mutex_unlock(&epc->mutex); 263 return; 264 } 265 266 static void *alloc_ep(int size, gfp_t gfp) 267 { 268 struct c4iw_ep_common *epc; 269 270 epc = kzalloc(size, gfp); 271 if (epc) { 272 kref_init(&epc->kref); 273 mutex_init(&epc->mutex); 274 c4iw_init_wr_wait(&epc->wr_wait); 275 } 276 PDBG("%s alloc ep %p\n", __func__, epc); 277 return epc; 278 } 279 280 void _c4iw_free_ep(struct kref *kref) 281 { 282 struct c4iw_ep *ep; 283 284 ep = container_of(kref, struct c4iw_ep, com.kref); 285 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 286 if (test_bit(QP_REFERENCED, &ep->com.flags)) 287 deref_qp(ep); 288 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 289 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 290 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 291 dst_release(ep->dst); 292 cxgb4_l2t_release(ep->l2t); 293 } 294 kfree(ep); 295 } 296 297 static void release_ep_resources(struct c4iw_ep *ep) 298 { 299 set_bit(RELEASE_RESOURCES, &ep->com.flags); 300 c4iw_put_ep(&ep->com); 301 } 302 303 static int status2errno(int status) 304 { 305 switch (status) { 306 case CPL_ERR_NONE: 307 return 0; 308 case CPL_ERR_CONN_RESET: 309 return -ECONNRESET; 310 case CPL_ERR_ARP_MISS: 311 return -EHOSTUNREACH; 312 case CPL_ERR_CONN_TIMEDOUT: 313 return -ETIMEDOUT; 314 case CPL_ERR_TCAM_FULL: 315 return -ENOMEM; 316 case CPL_ERR_CONN_EXIST: 317 return -EADDRINUSE; 318 default: 319 return -EIO; 320 } 321 } 322 323 /* 324 * Try and reuse skbs already allocated... 325 */ 326 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 327 { 328 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 329 skb_trim(skb, 0); 330 skb_get(skb); 331 skb_reset_transport_header(skb); 332 } else { 333 skb = alloc_skb(len, gfp); 334 } 335 t4_set_arp_err_handler(skb, NULL, NULL); 336 return skb; 337 } 338 339 static struct net_device *get_real_dev(struct net_device *egress_dev) 340 { 341 struct net_device *phys_dev = egress_dev; 342 if (egress_dev->priv_flags & IFF_802_1Q_VLAN) 343 phys_dev = vlan_dev_real_dev(egress_dev); 344 return phys_dev; 345 } 346 347 static int our_interface(struct c4iw_dev *dev, struct net_device *egress_dev) 348 { 349 int i; 350 351 egress_dev = get_real_dev(egress_dev); 352 for (i = 0; i < dev->rdev.lldi.nports; i++) 353 if (dev->rdev.lldi.ports[i] == egress_dev) 354 return 1; 355 return 0; 356 } 357 358 static struct dst_entry *find_route6(struct c4iw_dev *dev, __u8 *local_ip, 359 __u8 *peer_ip, __be16 local_port, 360 __be16 peer_port, u8 tos, 361 __u32 sin6_scope_id) 362 { 363 struct dst_entry *dst = NULL; 364 365 if (IS_ENABLED(CONFIG_IPV6)) { 366 struct flowi6 fl6; 367 368 memset(&fl6, 0, sizeof(fl6)); 369 memcpy(&fl6.daddr, peer_ip, 16); 370 memcpy(&fl6.saddr, local_ip, 16); 371 if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL) 372 fl6.flowi6_oif = sin6_scope_id; 373 dst = ip6_route_output(&init_net, NULL, &fl6); 374 if (!dst) 375 goto out; 376 if (!our_interface(dev, ip6_dst_idev(dst)->dev) && 377 !(ip6_dst_idev(dst)->dev->flags & IFF_LOOPBACK)) { 378 dst_release(dst); 379 dst = NULL; 380 } 381 } 382 383 out: 384 return dst; 385 } 386 387 static struct dst_entry *find_route(struct c4iw_dev *dev, __be32 local_ip, 388 __be32 peer_ip, __be16 local_port, 389 __be16 peer_port, u8 tos) 390 { 391 struct rtable *rt; 392 struct flowi4 fl4; 393 struct neighbour *n; 394 395 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 396 peer_port, local_port, IPPROTO_TCP, 397 tos, 0); 398 if (IS_ERR(rt)) 399 return NULL; 400 n = dst_neigh_lookup(&rt->dst, &peer_ip); 401 if (!n) 402 return NULL; 403 if (!our_interface(dev, n->dev) && 404 !(n->dev->flags & IFF_LOOPBACK)) { 405 dst_release(&rt->dst); 406 return NULL; 407 } 408 neigh_release(n); 409 return &rt->dst; 410 } 411 412 static void arp_failure_discard(void *handle, struct sk_buff *skb) 413 { 414 PDBG("%s c4iw_dev %p\n", __func__, handle); 415 kfree_skb(skb); 416 } 417 418 /* 419 * Handle an ARP failure for an active open. 420 */ 421 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 422 { 423 printk(KERN_ERR MOD "ARP failure duing connect\n"); 424 kfree_skb(skb); 425 } 426 427 /* 428 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 429 * and send it along. 430 */ 431 static void abort_arp_failure(void *handle, struct sk_buff *skb) 432 { 433 struct c4iw_rdev *rdev = handle; 434 struct cpl_abort_req *req = cplhdr(skb); 435 436 PDBG("%s rdev %p\n", __func__, rdev); 437 req->cmd = CPL_ABORT_NO_RST; 438 c4iw_ofld_send(rdev, skb); 439 } 440 441 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 442 { 443 unsigned int flowclen = 80; 444 struct fw_flowc_wr *flowc; 445 int i; 446 447 skb = get_skb(skb, flowclen, GFP_KERNEL); 448 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 449 450 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 451 FW_FLOWC_WR_NPARAMS(8)); 452 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 453 16)) | FW_WR_FLOWID(ep->hwtid)); 454 455 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 456 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 457 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 458 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 459 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 460 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 461 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 462 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 463 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 464 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 465 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 466 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 467 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 468 flowc->mnemval[6].val = cpu_to_be32(snd_win); 469 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 470 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 471 /* Pad WR to 16 byte boundary */ 472 flowc->mnemval[8].mnemonic = 0; 473 flowc->mnemval[8].val = 0; 474 for (i = 0; i < 9; i++) { 475 flowc->mnemval[i].r4[0] = 0; 476 flowc->mnemval[i].r4[1] = 0; 477 flowc->mnemval[i].r4[2] = 0; 478 } 479 480 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 481 c4iw_ofld_send(&ep->com.dev->rdev, skb); 482 } 483 484 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 485 { 486 struct cpl_close_con_req *req; 487 struct sk_buff *skb; 488 int wrlen = roundup(sizeof *req, 16); 489 490 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 491 skb = get_skb(NULL, wrlen, gfp); 492 if (!skb) { 493 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 494 return -ENOMEM; 495 } 496 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 497 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 498 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 499 memset(req, 0, wrlen); 500 INIT_TP_WR(req, ep->hwtid); 501 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 502 ep->hwtid)); 503 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 504 } 505 506 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 507 { 508 struct cpl_abort_req *req; 509 int wrlen = roundup(sizeof *req, 16); 510 511 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 512 skb = get_skb(skb, wrlen, gfp); 513 if (!skb) { 514 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 515 __func__); 516 return -ENOMEM; 517 } 518 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 519 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 520 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 521 memset(req, 0, wrlen); 522 INIT_TP_WR(req, ep->hwtid); 523 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 524 req->cmd = CPL_ABORT_SEND_RST; 525 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 526 } 527 528 static int send_connect(struct c4iw_ep *ep) 529 { 530 struct cpl_act_open_req *req; 531 struct cpl_t5_act_open_req *t5_req; 532 struct cpl_act_open_req6 *req6; 533 struct cpl_t5_act_open_req6 *t5_req6; 534 struct sk_buff *skb; 535 u64 opt0; 536 u32 opt2; 537 unsigned int mtu_idx; 538 int wscale; 539 int wrlen; 540 int sizev4 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 541 sizeof(struct cpl_act_open_req) : 542 sizeof(struct cpl_t5_act_open_req); 543 int sizev6 = is_t4(ep->com.dev->rdev.lldi.adapter_type) ? 544 sizeof(struct cpl_act_open_req6) : 545 sizeof(struct cpl_t5_act_open_req6); 546 struct sockaddr_in *la = (struct sockaddr_in *)&ep->com.local_addr; 547 struct sockaddr_in *ra = (struct sockaddr_in *)&ep->com.remote_addr; 548 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 549 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 550 551 wrlen = (ep->com.remote_addr.ss_family == AF_INET) ? 552 roundup(sizev4, 16) : 553 roundup(sizev6, 16); 554 555 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 556 557 skb = get_skb(NULL, wrlen, GFP_KERNEL); 558 if (!skb) { 559 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 560 __func__); 561 return -ENOMEM; 562 } 563 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 564 565 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 566 wscale = compute_wscale(rcv_win); 567 opt0 = (nocong ? NO_CONG(1) : 0) | 568 KEEP_ALIVE(1) | 569 DELACK(1) | 570 WND_SCALE(wscale) | 571 MSS_IDX(mtu_idx) | 572 L2T_IDX(ep->l2t->idx) | 573 TX_CHAN(ep->tx_chan) | 574 SMAC_SEL(ep->smac_idx) | 575 DSCP(ep->tos) | 576 ULP_MODE(ULP_MODE_TCPDDP) | 577 RCV_BUFSIZ(rcv_win>>10); 578 opt2 = RX_CHANNEL(0) | 579 CCTRL_ECN(enable_ecn) | 580 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 581 if (enable_tcp_timestamps) 582 opt2 |= TSTAMPS_EN(1); 583 if (enable_tcp_sack) 584 opt2 |= SACK_EN(1); 585 if (wscale && enable_tcp_window_scaling) 586 opt2 |= WND_SCALE_EN(1); 587 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 588 589 if (is_t4(ep->com.dev->rdev.lldi.adapter_type)) { 590 if (ep->com.remote_addr.ss_family == AF_INET) { 591 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 592 INIT_TP_WR(req, 0); 593 OPCODE_TID(req) = cpu_to_be32( 594 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 595 ((ep->rss_qid << 14) | ep->atid))); 596 req->local_port = la->sin_port; 597 req->peer_port = ra->sin_port; 598 req->local_ip = la->sin_addr.s_addr; 599 req->peer_ip = ra->sin_addr.s_addr; 600 req->opt0 = cpu_to_be64(opt0); 601 req->params = cpu_to_be32(cxgb4_select_ntuple( 602 ep->com.dev->rdev.lldi.ports[0], 603 ep->l2t)); 604 req->opt2 = cpu_to_be32(opt2); 605 } else { 606 req6 = (struct cpl_act_open_req6 *)skb_put(skb, wrlen); 607 608 INIT_TP_WR(req6, 0); 609 OPCODE_TID(req6) = cpu_to_be32( 610 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 611 ((ep->rss_qid<<14)|ep->atid))); 612 req6->local_port = la6->sin6_port; 613 req6->peer_port = ra6->sin6_port; 614 req6->local_ip_hi = *((__be64 *) 615 (la6->sin6_addr.s6_addr)); 616 req6->local_ip_lo = *((__be64 *) 617 (la6->sin6_addr.s6_addr + 8)); 618 req6->peer_ip_hi = *((__be64 *) 619 (ra6->sin6_addr.s6_addr)); 620 req6->peer_ip_lo = *((__be64 *) 621 (ra6->sin6_addr.s6_addr + 8)); 622 req6->opt0 = cpu_to_be64(opt0); 623 req6->params = cpu_to_be32(cxgb4_select_ntuple( 624 ep->com.dev->rdev.lldi.ports[0], 625 ep->l2t)); 626 req6->opt2 = cpu_to_be32(opt2); 627 } 628 } else { 629 if (ep->com.remote_addr.ss_family == AF_INET) { 630 t5_req = (struct cpl_t5_act_open_req *) 631 skb_put(skb, wrlen); 632 INIT_TP_WR(t5_req, 0); 633 OPCODE_TID(t5_req) = cpu_to_be32( 634 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 635 ((ep->rss_qid << 14) | ep->atid))); 636 t5_req->local_port = la->sin_port; 637 t5_req->peer_port = ra->sin_port; 638 t5_req->local_ip = la->sin_addr.s_addr; 639 t5_req->peer_ip = ra->sin_addr.s_addr; 640 t5_req->opt0 = cpu_to_be64(opt0); 641 t5_req->params = cpu_to_be64(V_FILTER_TUPLE( 642 cxgb4_select_ntuple( 643 ep->com.dev->rdev.lldi.ports[0], 644 ep->l2t))); 645 t5_req->opt2 = cpu_to_be32(opt2); 646 } else { 647 t5_req6 = (struct cpl_t5_act_open_req6 *) 648 skb_put(skb, wrlen); 649 INIT_TP_WR(t5_req6, 0); 650 OPCODE_TID(t5_req6) = cpu_to_be32( 651 MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 652 ((ep->rss_qid<<14)|ep->atid))); 653 t5_req6->local_port = la6->sin6_port; 654 t5_req6->peer_port = ra6->sin6_port; 655 t5_req6->local_ip_hi = *((__be64 *) 656 (la6->sin6_addr.s6_addr)); 657 t5_req6->local_ip_lo = *((__be64 *) 658 (la6->sin6_addr.s6_addr + 8)); 659 t5_req6->peer_ip_hi = *((__be64 *) 660 (ra6->sin6_addr.s6_addr)); 661 t5_req6->peer_ip_lo = *((__be64 *) 662 (ra6->sin6_addr.s6_addr + 8)); 663 t5_req6->opt0 = cpu_to_be64(opt0); 664 t5_req6->params = (__force __be64)cpu_to_be32( 665 cxgb4_select_ntuple( 666 ep->com.dev->rdev.lldi.ports[0], 667 ep->l2t)); 668 t5_req6->opt2 = cpu_to_be32(opt2); 669 } 670 } 671 672 set_bit(ACT_OPEN_REQ, &ep->com.history); 673 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 674 } 675 676 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 677 u8 mpa_rev_to_use) 678 { 679 int mpalen, wrlen; 680 struct fw_ofld_tx_data_wr *req; 681 struct mpa_message *mpa; 682 struct mpa_v2_conn_params mpa_v2_params; 683 684 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 685 686 BUG_ON(skb_cloned(skb)); 687 688 mpalen = sizeof(*mpa) + ep->plen; 689 if (mpa_rev_to_use == 2) 690 mpalen += sizeof(struct mpa_v2_conn_params); 691 wrlen = roundup(mpalen + sizeof *req, 16); 692 skb = get_skb(skb, wrlen, GFP_KERNEL); 693 if (!skb) { 694 connect_reply_upcall(ep, -ENOMEM); 695 return; 696 } 697 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 698 699 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 700 memset(req, 0, wrlen); 701 req->op_to_immdlen = cpu_to_be32( 702 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 703 FW_WR_COMPL(1) | 704 FW_WR_IMMDLEN(mpalen)); 705 req->flowid_len16 = cpu_to_be32( 706 FW_WR_FLOWID(ep->hwtid) | 707 FW_WR_LEN16(wrlen >> 4)); 708 req->plen = cpu_to_be32(mpalen); 709 req->tunnel_to_proxy = cpu_to_be32( 710 FW_OFLD_TX_DATA_WR_FLUSH(1) | 711 FW_OFLD_TX_DATA_WR_SHOVE(1)); 712 713 mpa = (struct mpa_message *)(req + 1); 714 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 715 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 716 (markers_enabled ? MPA_MARKERS : 0) | 717 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 718 mpa->private_data_size = htons(ep->plen); 719 mpa->revision = mpa_rev_to_use; 720 if (mpa_rev_to_use == 1) { 721 ep->tried_with_mpa_v1 = 1; 722 ep->retry_with_mpa_v1 = 0; 723 } 724 725 if (mpa_rev_to_use == 2) { 726 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 727 sizeof (struct mpa_v2_conn_params)); 728 mpa_v2_params.ird = htons((u16)ep->ird); 729 mpa_v2_params.ord = htons((u16)ep->ord); 730 731 if (peer2peer) { 732 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 733 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 734 mpa_v2_params.ord |= 735 htons(MPA_V2_RDMA_WRITE_RTR); 736 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 737 mpa_v2_params.ord |= 738 htons(MPA_V2_RDMA_READ_RTR); 739 } 740 memcpy(mpa->private_data, &mpa_v2_params, 741 sizeof(struct mpa_v2_conn_params)); 742 743 if (ep->plen) 744 memcpy(mpa->private_data + 745 sizeof(struct mpa_v2_conn_params), 746 ep->mpa_pkt + sizeof(*mpa), ep->plen); 747 } else 748 if (ep->plen) 749 memcpy(mpa->private_data, 750 ep->mpa_pkt + sizeof(*mpa), ep->plen); 751 752 /* 753 * Reference the mpa skb. This ensures the data area 754 * will remain in memory until the hw acks the tx. 755 * Function fw4_ack() will deref it. 756 */ 757 skb_get(skb); 758 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 759 BUG_ON(ep->mpa_skb); 760 ep->mpa_skb = skb; 761 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 762 start_ep_timer(ep); 763 __state_set(&ep->com, MPA_REQ_SENT); 764 ep->mpa_attr.initiator = 1; 765 ep->snd_seq += mpalen; 766 return; 767 } 768 769 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 770 { 771 int mpalen, wrlen; 772 struct fw_ofld_tx_data_wr *req; 773 struct mpa_message *mpa; 774 struct sk_buff *skb; 775 struct mpa_v2_conn_params mpa_v2_params; 776 777 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 778 779 mpalen = sizeof(*mpa) + plen; 780 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 781 mpalen += sizeof(struct mpa_v2_conn_params); 782 wrlen = roundup(mpalen + sizeof *req, 16); 783 784 skb = get_skb(NULL, wrlen, GFP_KERNEL); 785 if (!skb) { 786 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 787 return -ENOMEM; 788 } 789 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 790 791 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 792 memset(req, 0, wrlen); 793 req->op_to_immdlen = cpu_to_be32( 794 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 795 FW_WR_COMPL(1) | 796 FW_WR_IMMDLEN(mpalen)); 797 req->flowid_len16 = cpu_to_be32( 798 FW_WR_FLOWID(ep->hwtid) | 799 FW_WR_LEN16(wrlen >> 4)); 800 req->plen = cpu_to_be32(mpalen); 801 req->tunnel_to_proxy = cpu_to_be32( 802 FW_OFLD_TX_DATA_WR_FLUSH(1) | 803 FW_OFLD_TX_DATA_WR_SHOVE(1)); 804 805 mpa = (struct mpa_message *)(req + 1); 806 memset(mpa, 0, sizeof(*mpa)); 807 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 808 mpa->flags = MPA_REJECT; 809 mpa->revision = ep->mpa_attr.version; 810 mpa->private_data_size = htons(plen); 811 812 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 813 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 814 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 815 sizeof (struct mpa_v2_conn_params)); 816 mpa_v2_params.ird = htons(((u16)ep->ird) | 817 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 818 0)); 819 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 820 (p2p_type == 821 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 822 MPA_V2_RDMA_WRITE_RTR : p2p_type == 823 FW_RI_INIT_P2PTYPE_READ_REQ ? 824 MPA_V2_RDMA_READ_RTR : 0) : 0)); 825 memcpy(mpa->private_data, &mpa_v2_params, 826 sizeof(struct mpa_v2_conn_params)); 827 828 if (ep->plen) 829 memcpy(mpa->private_data + 830 sizeof(struct mpa_v2_conn_params), pdata, plen); 831 } else 832 if (plen) 833 memcpy(mpa->private_data, pdata, plen); 834 835 /* 836 * Reference the mpa skb again. This ensures the data area 837 * will remain in memory until the hw acks the tx. 838 * Function fw4_ack() will deref it. 839 */ 840 skb_get(skb); 841 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 842 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 843 BUG_ON(ep->mpa_skb); 844 ep->mpa_skb = skb; 845 ep->snd_seq += mpalen; 846 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 847 } 848 849 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 850 { 851 int mpalen, wrlen; 852 struct fw_ofld_tx_data_wr *req; 853 struct mpa_message *mpa; 854 struct sk_buff *skb; 855 struct mpa_v2_conn_params mpa_v2_params; 856 857 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 858 859 mpalen = sizeof(*mpa) + plen; 860 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 861 mpalen += sizeof(struct mpa_v2_conn_params); 862 wrlen = roundup(mpalen + sizeof *req, 16); 863 864 skb = get_skb(NULL, wrlen, GFP_KERNEL); 865 if (!skb) { 866 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 867 return -ENOMEM; 868 } 869 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 870 871 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 872 memset(req, 0, wrlen); 873 req->op_to_immdlen = cpu_to_be32( 874 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 875 FW_WR_COMPL(1) | 876 FW_WR_IMMDLEN(mpalen)); 877 req->flowid_len16 = cpu_to_be32( 878 FW_WR_FLOWID(ep->hwtid) | 879 FW_WR_LEN16(wrlen >> 4)); 880 req->plen = cpu_to_be32(mpalen); 881 req->tunnel_to_proxy = cpu_to_be32( 882 FW_OFLD_TX_DATA_WR_FLUSH(1) | 883 FW_OFLD_TX_DATA_WR_SHOVE(1)); 884 885 mpa = (struct mpa_message *)(req + 1); 886 memset(mpa, 0, sizeof(*mpa)); 887 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 888 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 889 (markers_enabled ? MPA_MARKERS : 0); 890 mpa->revision = ep->mpa_attr.version; 891 mpa->private_data_size = htons(plen); 892 893 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 894 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 895 mpa->private_data_size = htons(ntohs(mpa->private_data_size) + 896 sizeof (struct mpa_v2_conn_params)); 897 mpa_v2_params.ird = htons((u16)ep->ird); 898 mpa_v2_params.ord = htons((u16)ep->ord); 899 if (peer2peer && (ep->mpa_attr.p2p_type != 900 FW_RI_INIT_P2PTYPE_DISABLED)) { 901 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 902 903 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 904 mpa_v2_params.ord |= 905 htons(MPA_V2_RDMA_WRITE_RTR); 906 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 907 mpa_v2_params.ord |= 908 htons(MPA_V2_RDMA_READ_RTR); 909 } 910 911 memcpy(mpa->private_data, &mpa_v2_params, 912 sizeof(struct mpa_v2_conn_params)); 913 914 if (ep->plen) 915 memcpy(mpa->private_data + 916 sizeof(struct mpa_v2_conn_params), pdata, plen); 917 } else 918 if (plen) 919 memcpy(mpa->private_data, pdata, plen); 920 921 /* 922 * Reference the mpa skb. This ensures the data area 923 * will remain in memory until the hw acks the tx. 924 * Function fw4_ack() will deref it. 925 */ 926 skb_get(skb); 927 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 928 ep->mpa_skb = skb; 929 __state_set(&ep->com, MPA_REP_SENT); 930 ep->snd_seq += mpalen; 931 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 932 } 933 934 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 935 { 936 struct c4iw_ep *ep; 937 struct cpl_act_establish *req = cplhdr(skb); 938 unsigned int tid = GET_TID(req); 939 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 940 struct tid_info *t = dev->rdev.lldi.tids; 941 942 ep = lookup_atid(t, atid); 943 944 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 945 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 946 947 mutex_lock(&ep->com.mutex); 948 dst_confirm(ep->dst); 949 950 /* setup the hwtid for this connection */ 951 ep->hwtid = tid; 952 cxgb4_insert_tid(t, ep, tid); 953 insert_handle(dev, &dev->hwtid_idr, ep, ep->hwtid); 954 955 ep->snd_seq = be32_to_cpu(req->snd_isn); 956 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 957 958 set_emss(ep, ntohs(req->tcp_opt)); 959 960 /* dealloc the atid */ 961 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 962 cxgb4_free_atid(t, atid); 963 set_bit(ACT_ESTAB, &ep->com.history); 964 965 /* start MPA negotiation */ 966 send_flowc(ep, NULL); 967 if (ep->retry_with_mpa_v1) 968 send_mpa_req(ep, skb, 1); 969 else 970 send_mpa_req(ep, skb, mpa_rev); 971 mutex_unlock(&ep->com.mutex); 972 return 0; 973 } 974 975 static void close_complete_upcall(struct c4iw_ep *ep, int status) 976 { 977 struct iw_cm_event event; 978 979 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 980 memset(&event, 0, sizeof(event)); 981 event.event = IW_CM_EVENT_CLOSE; 982 event.status = status; 983 if (ep->com.cm_id) { 984 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 985 ep, ep->com.cm_id, ep->hwtid); 986 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 987 ep->com.cm_id->rem_ref(ep->com.cm_id); 988 ep->com.cm_id = NULL; 989 set_bit(CLOSE_UPCALL, &ep->com.history); 990 } 991 } 992 993 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 994 { 995 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 996 state_set(&ep->com, ABORTING); 997 set_bit(ABORT_CONN, &ep->com.history); 998 return send_abort(ep, skb, gfp); 999 } 1000 1001 static void peer_close_upcall(struct c4iw_ep *ep) 1002 { 1003 struct iw_cm_event event; 1004 1005 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1006 memset(&event, 0, sizeof(event)); 1007 event.event = IW_CM_EVENT_DISCONNECT; 1008 if (ep->com.cm_id) { 1009 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 1010 ep, ep->com.cm_id, ep->hwtid); 1011 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1012 set_bit(DISCONN_UPCALL, &ep->com.history); 1013 } 1014 } 1015 1016 static void peer_abort_upcall(struct c4iw_ep *ep) 1017 { 1018 struct iw_cm_event event; 1019 1020 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1021 memset(&event, 0, sizeof(event)); 1022 event.event = IW_CM_EVENT_CLOSE; 1023 event.status = -ECONNRESET; 1024 if (ep->com.cm_id) { 1025 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 1026 ep->com.cm_id, ep->hwtid); 1027 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1028 ep->com.cm_id->rem_ref(ep->com.cm_id); 1029 ep->com.cm_id = NULL; 1030 set_bit(ABORT_UPCALL, &ep->com.history); 1031 } 1032 } 1033 1034 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 1035 { 1036 struct iw_cm_event event; 1037 1038 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 1039 memset(&event, 0, sizeof(event)); 1040 event.event = IW_CM_EVENT_CONNECT_REPLY; 1041 event.status = status; 1042 memcpy(&event.local_addr, &ep->com.local_addr, 1043 sizeof(ep->com.local_addr)); 1044 memcpy(&event.remote_addr, &ep->com.remote_addr, 1045 sizeof(ep->com.remote_addr)); 1046 1047 if ((status == 0) || (status == -ECONNREFUSED)) { 1048 if (!ep->tried_with_mpa_v1) { 1049 /* this means MPA_v2 is used */ 1050 event.private_data_len = ep->plen - 1051 sizeof(struct mpa_v2_conn_params); 1052 event.private_data = ep->mpa_pkt + 1053 sizeof(struct mpa_message) + 1054 sizeof(struct mpa_v2_conn_params); 1055 } else { 1056 /* this means MPA_v1 is used */ 1057 event.private_data_len = ep->plen; 1058 event.private_data = ep->mpa_pkt + 1059 sizeof(struct mpa_message); 1060 } 1061 } 1062 1063 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 1064 ep->hwtid, status); 1065 set_bit(CONN_RPL_UPCALL, &ep->com.history); 1066 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1067 1068 if (status < 0) { 1069 ep->com.cm_id->rem_ref(ep->com.cm_id); 1070 ep->com.cm_id = NULL; 1071 } 1072 } 1073 1074 static int connect_request_upcall(struct c4iw_ep *ep) 1075 { 1076 struct iw_cm_event event; 1077 int ret; 1078 1079 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1080 memset(&event, 0, sizeof(event)); 1081 event.event = IW_CM_EVENT_CONNECT_REQUEST; 1082 memcpy(&event.local_addr, &ep->com.local_addr, 1083 sizeof(ep->com.local_addr)); 1084 memcpy(&event.remote_addr, &ep->com.remote_addr, 1085 sizeof(ep->com.remote_addr)); 1086 event.provider_data = ep; 1087 if (!ep->tried_with_mpa_v1) { 1088 /* this means MPA_v2 is used */ 1089 event.ord = ep->ord; 1090 event.ird = ep->ird; 1091 event.private_data_len = ep->plen - 1092 sizeof(struct mpa_v2_conn_params); 1093 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 1094 sizeof(struct mpa_v2_conn_params); 1095 } else { 1096 /* this means MPA_v1 is used. Send max supported */ 1097 event.ord = c4iw_max_read_depth; 1098 event.ird = c4iw_max_read_depth; 1099 event.private_data_len = ep->plen; 1100 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 1101 } 1102 c4iw_get_ep(&ep->com); 1103 ret = ep->parent_ep->com.cm_id->event_handler(ep->parent_ep->com.cm_id, 1104 &event); 1105 if (ret) 1106 c4iw_put_ep(&ep->com); 1107 set_bit(CONNREQ_UPCALL, &ep->com.history); 1108 c4iw_put_ep(&ep->parent_ep->com); 1109 return ret; 1110 } 1111 1112 static void established_upcall(struct c4iw_ep *ep) 1113 { 1114 struct iw_cm_event event; 1115 1116 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1117 memset(&event, 0, sizeof(event)); 1118 event.event = IW_CM_EVENT_ESTABLISHED; 1119 event.ird = ep->ird; 1120 event.ord = ep->ord; 1121 if (ep->com.cm_id) { 1122 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1123 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 1124 set_bit(ESTAB_UPCALL, &ep->com.history); 1125 } 1126 } 1127 1128 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 1129 { 1130 struct cpl_rx_data_ack *req; 1131 struct sk_buff *skb; 1132 int wrlen = roundup(sizeof *req, 16); 1133 1134 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 1135 skb = get_skb(NULL, wrlen, GFP_KERNEL); 1136 if (!skb) { 1137 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 1138 return 0; 1139 } 1140 1141 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 1142 memset(req, 0, wrlen); 1143 INIT_TP_WR(req, ep->hwtid); 1144 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 1145 ep->hwtid)); 1146 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 1147 F_RX_DACK_CHANGE | 1148 V_RX_DACK_MODE(dack_mode)); 1149 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 1150 c4iw_ofld_send(&ep->com.dev->rdev, skb); 1151 return credits; 1152 } 1153 1154 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 1155 { 1156 struct mpa_message *mpa; 1157 struct mpa_v2_conn_params *mpa_v2_params; 1158 u16 plen; 1159 u16 resp_ird, resp_ord; 1160 u8 rtr_mismatch = 0, insuff_ird = 0; 1161 struct c4iw_qp_attributes attrs; 1162 enum c4iw_qp_attr_mask mask; 1163 int err; 1164 1165 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1166 1167 /* 1168 * Stop mpa timer. If it expired, then the state has 1169 * changed and we bail since ep_timeout already aborted 1170 * the connection. 1171 */ 1172 stop_ep_timer(ep); 1173 if (ep->com.state != MPA_REQ_SENT) 1174 return; 1175 1176 /* 1177 * If we get more than the supported amount of private data 1178 * then we must fail this connection. 1179 */ 1180 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1181 err = -EINVAL; 1182 goto err; 1183 } 1184 1185 /* 1186 * copy the new data into our accumulation buffer. 1187 */ 1188 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1189 skb->len); 1190 ep->mpa_pkt_len += skb->len; 1191 1192 /* 1193 * if we don't even have the mpa message, then bail. 1194 */ 1195 if (ep->mpa_pkt_len < sizeof(*mpa)) 1196 return; 1197 mpa = (struct mpa_message *) ep->mpa_pkt; 1198 1199 /* Validate MPA header. */ 1200 if (mpa->revision > mpa_rev) { 1201 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1202 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1203 err = -EPROTO; 1204 goto err; 1205 } 1206 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1207 err = -EPROTO; 1208 goto err; 1209 } 1210 1211 plen = ntohs(mpa->private_data_size); 1212 1213 /* 1214 * Fail if there's too much private data. 1215 */ 1216 if (plen > MPA_MAX_PRIVATE_DATA) { 1217 err = -EPROTO; 1218 goto err; 1219 } 1220 1221 /* 1222 * If plen does not account for pkt size 1223 */ 1224 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1225 err = -EPROTO; 1226 goto err; 1227 } 1228 1229 ep->plen = (u8) plen; 1230 1231 /* 1232 * If we don't have all the pdata yet, then bail. 1233 * We'll continue process when more data arrives. 1234 */ 1235 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1236 return; 1237 1238 if (mpa->flags & MPA_REJECT) { 1239 err = -ECONNREFUSED; 1240 goto err; 1241 } 1242 1243 /* 1244 * If we get here we have accumulated the entire mpa 1245 * start reply message including private data. And 1246 * the MPA header is valid. 1247 */ 1248 __state_set(&ep->com, FPDU_MODE); 1249 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1250 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1251 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1252 ep->mpa_attr.version = mpa->revision; 1253 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1254 1255 if (mpa->revision == 2) { 1256 ep->mpa_attr.enhanced_rdma_conn = 1257 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1258 if (ep->mpa_attr.enhanced_rdma_conn) { 1259 mpa_v2_params = (struct mpa_v2_conn_params *) 1260 (ep->mpa_pkt + sizeof(*mpa)); 1261 resp_ird = ntohs(mpa_v2_params->ird) & 1262 MPA_V2_IRD_ORD_MASK; 1263 resp_ord = ntohs(mpa_v2_params->ord) & 1264 MPA_V2_IRD_ORD_MASK; 1265 1266 /* 1267 * This is a double-check. Ideally, below checks are 1268 * not required since ird/ord stuff has been taken 1269 * care of in c4iw_accept_cr 1270 */ 1271 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1272 err = -ENOMEM; 1273 ep->ird = resp_ord; 1274 ep->ord = resp_ird; 1275 insuff_ird = 1; 1276 } 1277 1278 if (ntohs(mpa_v2_params->ird) & 1279 MPA_V2_PEER2PEER_MODEL) { 1280 if (ntohs(mpa_v2_params->ord) & 1281 MPA_V2_RDMA_WRITE_RTR) 1282 ep->mpa_attr.p2p_type = 1283 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1284 else if (ntohs(mpa_v2_params->ord) & 1285 MPA_V2_RDMA_READ_RTR) 1286 ep->mpa_attr.p2p_type = 1287 FW_RI_INIT_P2PTYPE_READ_REQ; 1288 } 1289 } 1290 } else if (mpa->revision == 1) 1291 if (peer2peer) 1292 ep->mpa_attr.p2p_type = p2p_type; 1293 1294 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1295 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1296 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1297 ep->mpa_attr.recv_marker_enabled, 1298 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1299 ep->mpa_attr.p2p_type, p2p_type); 1300 1301 /* 1302 * If responder's RTR does not match with that of initiator, assign 1303 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1304 * generated when moving QP to RTS state. 1305 * A TERM message will be sent after QP has moved to RTS state 1306 */ 1307 if ((ep->mpa_attr.version == 2) && peer2peer && 1308 (ep->mpa_attr.p2p_type != p2p_type)) { 1309 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1310 rtr_mismatch = 1; 1311 } 1312 1313 attrs.mpa_attr = ep->mpa_attr; 1314 attrs.max_ird = ep->ird; 1315 attrs.max_ord = ep->ord; 1316 attrs.llp_stream_handle = ep; 1317 attrs.next_state = C4IW_QP_STATE_RTS; 1318 1319 mask = C4IW_QP_ATTR_NEXT_STATE | 1320 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1321 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1322 1323 /* bind QP and TID with INIT_WR */ 1324 err = c4iw_modify_qp(ep->com.qp->rhp, 1325 ep->com.qp, mask, &attrs, 1); 1326 if (err) 1327 goto err; 1328 1329 /* 1330 * If responder's RTR requirement did not match with what initiator 1331 * supports, generate TERM message 1332 */ 1333 if (rtr_mismatch) { 1334 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1335 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1336 attrs.ecode = MPA_NOMATCH_RTR; 1337 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1338 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1339 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1340 err = -ENOMEM; 1341 goto out; 1342 } 1343 1344 /* 1345 * Generate TERM if initiator IRD is not sufficient for responder 1346 * provided ORD. Currently, we do the same behaviour even when 1347 * responder provided IRD is also not sufficient as regards to 1348 * initiator ORD. 1349 */ 1350 if (insuff_ird) { 1351 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1352 __func__); 1353 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1354 attrs.ecode = MPA_INSUFF_IRD; 1355 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1356 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1357 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1358 err = -ENOMEM; 1359 goto out; 1360 } 1361 goto out; 1362 err: 1363 __state_set(&ep->com, ABORTING); 1364 send_abort(ep, skb, GFP_KERNEL); 1365 out: 1366 connect_reply_upcall(ep, err); 1367 return; 1368 } 1369 1370 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1371 { 1372 struct mpa_message *mpa; 1373 struct mpa_v2_conn_params *mpa_v2_params; 1374 u16 plen; 1375 1376 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1377 1378 if (ep->com.state != MPA_REQ_WAIT) 1379 return; 1380 1381 /* 1382 * If we get more than the supported amount of private data 1383 * then we must fail this connection. 1384 */ 1385 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1386 stop_ep_timer(ep); 1387 abort_connection(ep, skb, GFP_KERNEL); 1388 return; 1389 } 1390 1391 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1392 1393 /* 1394 * Copy the new data into our accumulation buffer. 1395 */ 1396 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1397 skb->len); 1398 ep->mpa_pkt_len += skb->len; 1399 1400 /* 1401 * If we don't even have the mpa message, then bail. 1402 * We'll continue process when more data arrives. 1403 */ 1404 if (ep->mpa_pkt_len < sizeof(*mpa)) 1405 return; 1406 1407 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1408 mpa = (struct mpa_message *) ep->mpa_pkt; 1409 1410 /* 1411 * Validate MPA Header. 1412 */ 1413 if (mpa->revision > mpa_rev) { 1414 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1415 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1416 stop_ep_timer(ep); 1417 abort_connection(ep, skb, GFP_KERNEL); 1418 return; 1419 } 1420 1421 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1422 stop_ep_timer(ep); 1423 abort_connection(ep, skb, GFP_KERNEL); 1424 return; 1425 } 1426 1427 plen = ntohs(mpa->private_data_size); 1428 1429 /* 1430 * Fail if there's too much private data. 1431 */ 1432 if (plen > MPA_MAX_PRIVATE_DATA) { 1433 stop_ep_timer(ep); 1434 abort_connection(ep, skb, GFP_KERNEL); 1435 return; 1436 } 1437 1438 /* 1439 * If plen does not account for pkt size 1440 */ 1441 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1442 stop_ep_timer(ep); 1443 abort_connection(ep, skb, GFP_KERNEL); 1444 return; 1445 } 1446 ep->plen = (u8) plen; 1447 1448 /* 1449 * If we don't have all the pdata yet, then bail. 1450 */ 1451 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1452 return; 1453 1454 /* 1455 * If we get here we have accumulated the entire mpa 1456 * start reply message including private data. 1457 */ 1458 ep->mpa_attr.initiator = 0; 1459 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1460 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1461 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1462 ep->mpa_attr.version = mpa->revision; 1463 if (mpa->revision == 1) 1464 ep->tried_with_mpa_v1 = 1; 1465 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1466 1467 if (mpa->revision == 2) { 1468 ep->mpa_attr.enhanced_rdma_conn = 1469 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1470 if (ep->mpa_attr.enhanced_rdma_conn) { 1471 mpa_v2_params = (struct mpa_v2_conn_params *) 1472 (ep->mpa_pkt + sizeof(*mpa)); 1473 ep->ird = ntohs(mpa_v2_params->ird) & 1474 MPA_V2_IRD_ORD_MASK; 1475 ep->ord = ntohs(mpa_v2_params->ord) & 1476 MPA_V2_IRD_ORD_MASK; 1477 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1478 if (peer2peer) { 1479 if (ntohs(mpa_v2_params->ord) & 1480 MPA_V2_RDMA_WRITE_RTR) 1481 ep->mpa_attr.p2p_type = 1482 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1483 else if (ntohs(mpa_v2_params->ord) & 1484 MPA_V2_RDMA_READ_RTR) 1485 ep->mpa_attr.p2p_type = 1486 FW_RI_INIT_P2PTYPE_READ_REQ; 1487 } 1488 } 1489 } else if (mpa->revision == 1) 1490 if (peer2peer) 1491 ep->mpa_attr.p2p_type = p2p_type; 1492 1493 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1494 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1495 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1496 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1497 ep->mpa_attr.p2p_type); 1498 1499 __state_set(&ep->com, MPA_REQ_RCVD); 1500 stop_ep_timer(ep); 1501 1502 /* drive upcall */ 1503 mutex_lock(&ep->parent_ep->com.mutex); 1504 if (ep->parent_ep->com.state != DEAD) { 1505 if (connect_request_upcall(ep)) 1506 abort_connection(ep, skb, GFP_KERNEL); 1507 } else { 1508 abort_connection(ep, skb, GFP_KERNEL); 1509 } 1510 mutex_unlock(&ep->parent_ep->com.mutex); 1511 return; 1512 } 1513 1514 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1515 { 1516 struct c4iw_ep *ep; 1517 struct cpl_rx_data *hdr = cplhdr(skb); 1518 unsigned int dlen = ntohs(hdr->len); 1519 unsigned int tid = GET_TID(hdr); 1520 struct tid_info *t = dev->rdev.lldi.tids; 1521 __u8 status = hdr->status; 1522 1523 ep = lookup_tid(t, tid); 1524 if (!ep) 1525 return 0; 1526 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1527 skb_pull(skb, sizeof(*hdr)); 1528 skb_trim(skb, dlen); 1529 mutex_lock(&ep->com.mutex); 1530 1531 /* update RX credits */ 1532 update_rx_credits(ep, dlen); 1533 1534 switch (ep->com.state) { 1535 case MPA_REQ_SENT: 1536 ep->rcv_seq += dlen; 1537 process_mpa_reply(ep, skb); 1538 break; 1539 case MPA_REQ_WAIT: 1540 ep->rcv_seq += dlen; 1541 process_mpa_request(ep, skb); 1542 break; 1543 case FPDU_MODE: { 1544 struct c4iw_qp_attributes attrs; 1545 BUG_ON(!ep->com.qp); 1546 if (status) 1547 pr_err("%s Unexpected streaming data." \ 1548 " qpid %u ep %p state %d tid %u status %d\n", 1549 __func__, ep->com.qp->wq.sq.qid, ep, 1550 ep->com.state, ep->hwtid, status); 1551 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1552 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1553 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1554 break; 1555 } 1556 default: 1557 break; 1558 } 1559 mutex_unlock(&ep->com.mutex); 1560 return 0; 1561 } 1562 1563 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1564 { 1565 struct c4iw_ep *ep; 1566 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1567 int release = 0; 1568 unsigned int tid = GET_TID(rpl); 1569 struct tid_info *t = dev->rdev.lldi.tids; 1570 1571 ep = lookup_tid(t, tid); 1572 if (!ep) { 1573 printk(KERN_WARNING MOD "Abort rpl to freed endpoint\n"); 1574 return 0; 1575 } 1576 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1577 mutex_lock(&ep->com.mutex); 1578 switch (ep->com.state) { 1579 case ABORTING: 1580 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1581 __state_set(&ep->com, DEAD); 1582 release = 1; 1583 break; 1584 default: 1585 printk(KERN_ERR "%s ep %p state %d\n", 1586 __func__, ep, ep->com.state); 1587 break; 1588 } 1589 mutex_unlock(&ep->com.mutex); 1590 1591 if (release) 1592 release_ep_resources(ep); 1593 return 0; 1594 } 1595 1596 static void send_fw_act_open_req(struct c4iw_ep *ep, unsigned int atid) 1597 { 1598 struct sk_buff *skb; 1599 struct fw_ofld_connection_wr *req; 1600 unsigned int mtu_idx; 1601 int wscale; 1602 struct sockaddr_in *sin; 1603 1604 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1605 req = (struct fw_ofld_connection_wr *)__skb_put(skb, sizeof(*req)); 1606 memset(req, 0, sizeof(*req)); 1607 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR)); 1608 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 1609 req->le.filter = cpu_to_be32(cxgb4_select_ntuple( 1610 ep->com.dev->rdev.lldi.ports[0], 1611 ep->l2t)); 1612 sin = (struct sockaddr_in *)&ep->com.local_addr; 1613 req->le.lport = sin->sin_port; 1614 req->le.u.ipv4.lip = sin->sin_addr.s_addr; 1615 sin = (struct sockaddr_in *)&ep->com.remote_addr; 1616 req->le.pport = sin->sin_port; 1617 req->le.u.ipv4.pip = sin->sin_addr.s_addr; 1618 req->tcb.t_state_to_astid = 1619 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_SENT) | 1620 V_FW_OFLD_CONNECTION_WR_ASTID(atid)); 1621 req->tcb.cplrxdataack_cplpassacceptrpl = 1622 htons(F_FW_OFLD_CONNECTION_WR_CPLRXDATAACK); 1623 req->tcb.tx_max = (__force __be32) jiffies; 1624 req->tcb.rcv_adv = htons(1); 1625 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1626 wscale = compute_wscale(rcv_win); 1627 req->tcb.opt0 = (__force __be64) (TCAM_BYPASS(1) | 1628 (nocong ? NO_CONG(1) : 0) | 1629 KEEP_ALIVE(1) | 1630 DELACK(1) | 1631 WND_SCALE(wscale) | 1632 MSS_IDX(mtu_idx) | 1633 L2T_IDX(ep->l2t->idx) | 1634 TX_CHAN(ep->tx_chan) | 1635 SMAC_SEL(ep->smac_idx) | 1636 DSCP(ep->tos) | 1637 ULP_MODE(ULP_MODE_TCPDDP) | 1638 RCV_BUFSIZ(rcv_win >> 10)); 1639 req->tcb.opt2 = (__force __be32) (PACE(1) | 1640 TX_QUEUE(ep->com.dev->rdev.lldi.tx_modq[ep->tx_chan]) | 1641 RX_CHANNEL(0) | 1642 CCTRL_ECN(enable_ecn) | 1643 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid)); 1644 if (enable_tcp_timestamps) 1645 req->tcb.opt2 |= (__force __be32) TSTAMPS_EN(1); 1646 if (enable_tcp_sack) 1647 req->tcb.opt2 |= (__force __be32) SACK_EN(1); 1648 if (wscale && enable_tcp_window_scaling) 1649 req->tcb.opt2 |= (__force __be32) WND_SCALE_EN(1); 1650 req->tcb.opt0 = cpu_to_be64((__force u64) req->tcb.opt0); 1651 req->tcb.opt2 = cpu_to_be32((__force u32) req->tcb.opt2); 1652 set_wr_txq(skb, CPL_PRIORITY_CONTROL, ep->ctrlq_idx); 1653 set_bit(ACT_OFLD_CONN, &ep->com.history); 1654 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1655 } 1656 1657 /* 1658 * Return whether a failed active open has allocated a TID 1659 */ 1660 static inline int act_open_has_tid(int status) 1661 { 1662 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1663 status != CPL_ERR_ARP_MISS; 1664 } 1665 1666 /* Returns whether a CPL status conveys negative advice. 1667 */ 1668 static int is_neg_adv(unsigned int status) 1669 { 1670 return status == CPL_ERR_RTX_NEG_ADVICE || 1671 status == CPL_ERR_PERSIST_NEG_ADVICE || 1672 status == CPL_ERR_KEEPALV_NEG_ADVICE; 1673 } 1674 1675 #define ACT_OPEN_RETRY_COUNT 2 1676 1677 static int import_ep(struct c4iw_ep *ep, int iptype, __u8 *peer_ip, 1678 struct dst_entry *dst, struct c4iw_dev *cdev, 1679 bool clear_mpa_v1) 1680 { 1681 struct neighbour *n; 1682 int err, step; 1683 struct net_device *pdev; 1684 1685 n = dst_neigh_lookup(dst, peer_ip); 1686 if (!n) 1687 return -ENODEV; 1688 1689 rcu_read_lock(); 1690 err = -ENOMEM; 1691 if (n->dev->flags & IFF_LOOPBACK) { 1692 if (iptype == 4) 1693 pdev = ip_dev_find(&init_net, *(__be32 *)peer_ip); 1694 else if (IS_ENABLED(CONFIG_IPV6)) 1695 for_each_netdev(&init_net, pdev) { 1696 if (ipv6_chk_addr(&init_net, 1697 (struct in6_addr *)peer_ip, 1698 pdev, 1)) 1699 break; 1700 } 1701 else 1702 pdev = NULL; 1703 1704 if (!pdev) { 1705 err = -ENODEV; 1706 goto out; 1707 } 1708 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1709 n, pdev, 0); 1710 if (!ep->l2t) 1711 goto out; 1712 ep->mtu = pdev->mtu; 1713 ep->tx_chan = cxgb4_port_chan(pdev); 1714 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1715 step = cdev->rdev.lldi.ntxq / 1716 cdev->rdev.lldi.nchan; 1717 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1718 step = cdev->rdev.lldi.nrxq / 1719 cdev->rdev.lldi.nchan; 1720 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1721 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1722 cxgb4_port_idx(pdev) * step]; 1723 dev_put(pdev); 1724 } else { 1725 pdev = get_real_dev(n->dev); 1726 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1727 n, pdev, 0); 1728 if (!ep->l2t) 1729 goto out; 1730 ep->mtu = dst_mtu(dst); 1731 ep->tx_chan = cxgb4_port_chan(n->dev); 1732 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1733 step = cdev->rdev.lldi.ntxq / 1734 cdev->rdev.lldi.nchan; 1735 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1736 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1737 step = cdev->rdev.lldi.nrxq / 1738 cdev->rdev.lldi.nchan; 1739 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1740 cxgb4_port_idx(n->dev) * step]; 1741 1742 if (clear_mpa_v1) { 1743 ep->retry_with_mpa_v1 = 0; 1744 ep->tried_with_mpa_v1 = 0; 1745 } 1746 } 1747 err = 0; 1748 out: 1749 rcu_read_unlock(); 1750 1751 neigh_release(n); 1752 1753 return err; 1754 } 1755 1756 static int c4iw_reconnect(struct c4iw_ep *ep) 1757 { 1758 int err = 0; 1759 struct sockaddr_in *laddr = (struct sockaddr_in *) 1760 &ep->com.cm_id->local_addr; 1761 struct sockaddr_in *raddr = (struct sockaddr_in *) 1762 &ep->com.cm_id->remote_addr; 1763 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *) 1764 &ep->com.cm_id->local_addr; 1765 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 1766 &ep->com.cm_id->remote_addr; 1767 int iptype; 1768 __u8 *ra; 1769 1770 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1771 init_timer(&ep->timer); 1772 1773 /* 1774 * Allocate an active TID to initiate a TCP connection. 1775 */ 1776 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1777 if (ep->atid == -1) { 1778 pr_err("%s - cannot alloc atid.\n", __func__); 1779 err = -ENOMEM; 1780 goto fail2; 1781 } 1782 insert_handle(ep->com.dev, &ep->com.dev->atid_idr, ep, ep->atid); 1783 1784 /* find a route */ 1785 if (ep->com.cm_id->local_addr.ss_family == AF_INET) { 1786 ep->dst = find_route(ep->com.dev, laddr->sin_addr.s_addr, 1787 raddr->sin_addr.s_addr, laddr->sin_port, 1788 raddr->sin_port, 0); 1789 iptype = 4; 1790 ra = (__u8 *)&raddr->sin_addr; 1791 } else { 1792 ep->dst = find_route6(ep->com.dev, laddr6->sin6_addr.s6_addr, 1793 raddr6->sin6_addr.s6_addr, 1794 laddr6->sin6_port, raddr6->sin6_port, 0, 1795 raddr6->sin6_scope_id); 1796 iptype = 6; 1797 ra = (__u8 *)&raddr6->sin6_addr; 1798 } 1799 if (!ep->dst) { 1800 pr_err("%s - cannot find route.\n", __func__); 1801 err = -EHOSTUNREACH; 1802 goto fail3; 1803 } 1804 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, false); 1805 if (err) { 1806 pr_err("%s - cannot alloc l2e.\n", __func__); 1807 goto fail4; 1808 } 1809 1810 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1811 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1812 ep->l2t->idx); 1813 1814 state_set(&ep->com, CONNECTING); 1815 ep->tos = 0; 1816 1817 /* send connect request to rnic */ 1818 err = send_connect(ep); 1819 if (!err) 1820 goto out; 1821 1822 cxgb4_l2t_release(ep->l2t); 1823 fail4: 1824 dst_release(ep->dst); 1825 fail3: 1826 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 1827 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1828 fail2: 1829 /* 1830 * remember to send notification to upper layer. 1831 * We are in here so the upper layer is not aware that this is 1832 * re-connect attempt and so, upper layer is still waiting for 1833 * response of 1st connect request. 1834 */ 1835 connect_reply_upcall(ep, -ECONNRESET); 1836 c4iw_put_ep(&ep->com); 1837 out: 1838 return err; 1839 } 1840 1841 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1842 { 1843 struct c4iw_ep *ep; 1844 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1845 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1846 ntohl(rpl->atid_status))); 1847 struct tid_info *t = dev->rdev.lldi.tids; 1848 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1849 struct sockaddr_in *la; 1850 struct sockaddr_in *ra; 1851 struct sockaddr_in6 *la6; 1852 struct sockaddr_in6 *ra6; 1853 1854 ep = lookup_atid(t, atid); 1855 la = (struct sockaddr_in *)&ep->com.local_addr; 1856 ra = (struct sockaddr_in *)&ep->com.remote_addr; 1857 la6 = (struct sockaddr_in6 *)&ep->com.local_addr; 1858 ra6 = (struct sockaddr_in6 *)&ep->com.remote_addr; 1859 1860 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1861 status, status2errno(status)); 1862 1863 if (is_neg_adv(status)) { 1864 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1865 atid); 1866 return 0; 1867 } 1868 1869 set_bit(ACT_OPEN_RPL, &ep->com.history); 1870 1871 /* 1872 * Log interesting failures. 1873 */ 1874 switch (status) { 1875 case CPL_ERR_CONN_RESET: 1876 case CPL_ERR_CONN_TIMEDOUT: 1877 break; 1878 case CPL_ERR_TCAM_FULL: 1879 mutex_lock(&dev->rdev.stats.lock); 1880 dev->rdev.stats.tcam_full++; 1881 mutex_unlock(&dev->rdev.stats.lock); 1882 if (ep->com.local_addr.ss_family == AF_INET && 1883 dev->rdev.lldi.enable_fw_ofld_conn) { 1884 send_fw_act_open_req(ep, 1885 GET_TID_TID(GET_AOPEN_ATID( 1886 ntohl(rpl->atid_status)))); 1887 return 0; 1888 } 1889 break; 1890 case CPL_ERR_CONN_EXIST: 1891 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 1892 set_bit(ACT_RETRY_INUSE, &ep->com.history); 1893 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, 1894 atid); 1895 cxgb4_free_atid(t, atid); 1896 dst_release(ep->dst); 1897 cxgb4_l2t_release(ep->l2t); 1898 c4iw_reconnect(ep); 1899 return 0; 1900 } 1901 break; 1902 default: 1903 if (ep->com.local_addr.ss_family == AF_INET) { 1904 pr_info("Active open failure - atid %u status %u errno %d %pI4:%u->%pI4:%u\n", 1905 atid, status, status2errno(status), 1906 &la->sin_addr.s_addr, ntohs(la->sin_port), 1907 &ra->sin_addr.s_addr, ntohs(ra->sin_port)); 1908 } else { 1909 pr_info("Active open failure - atid %u status %u errno %d %pI6:%u->%pI6:%u\n", 1910 atid, status, status2errno(status), 1911 la6->sin6_addr.s6_addr, ntohs(la6->sin6_port), 1912 ra6->sin6_addr.s6_addr, ntohs(ra6->sin6_port)); 1913 } 1914 break; 1915 } 1916 1917 connect_reply_upcall(ep, status2errno(status)); 1918 state_set(&ep->com, DEAD); 1919 1920 if (status && act_open_has_tid(status)) 1921 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1922 1923 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, atid); 1924 cxgb4_free_atid(t, atid); 1925 dst_release(ep->dst); 1926 cxgb4_l2t_release(ep->l2t); 1927 c4iw_put_ep(&ep->com); 1928 1929 return 0; 1930 } 1931 1932 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1933 { 1934 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1935 struct tid_info *t = dev->rdev.lldi.tids; 1936 unsigned int stid = GET_TID(rpl); 1937 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1938 1939 if (!ep) { 1940 PDBG("%s stid %d lookup failure!\n", __func__, stid); 1941 goto out; 1942 } 1943 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1944 rpl->status, status2errno(rpl->status)); 1945 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1946 1947 out: 1948 return 0; 1949 } 1950 1951 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1952 { 1953 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1954 struct tid_info *t = dev->rdev.lldi.tids; 1955 unsigned int stid = GET_TID(rpl); 1956 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1957 1958 PDBG("%s ep %p\n", __func__, ep); 1959 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1960 return 0; 1961 } 1962 1963 static void accept_cr(struct c4iw_ep *ep, struct sk_buff *skb, 1964 struct cpl_pass_accept_req *req) 1965 { 1966 struct cpl_pass_accept_rpl *rpl; 1967 unsigned int mtu_idx; 1968 u64 opt0; 1969 u32 opt2; 1970 int wscale; 1971 1972 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1973 BUG_ON(skb_cloned(skb)); 1974 skb_trim(skb, sizeof(*rpl)); 1975 skb_get(skb); 1976 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1977 wscale = compute_wscale(rcv_win); 1978 opt0 = (nocong ? NO_CONG(1) : 0) | 1979 KEEP_ALIVE(1) | 1980 DELACK(1) | 1981 WND_SCALE(wscale) | 1982 MSS_IDX(mtu_idx) | 1983 L2T_IDX(ep->l2t->idx) | 1984 TX_CHAN(ep->tx_chan) | 1985 SMAC_SEL(ep->smac_idx) | 1986 DSCP(ep->tos >> 2) | 1987 ULP_MODE(ULP_MODE_TCPDDP) | 1988 RCV_BUFSIZ(rcv_win>>10); 1989 opt2 = RX_CHANNEL(0) | 1990 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1991 1992 if (enable_tcp_timestamps && req->tcpopt.tstamp) 1993 opt2 |= TSTAMPS_EN(1); 1994 if (enable_tcp_sack && req->tcpopt.sack) 1995 opt2 |= SACK_EN(1); 1996 if (wscale && enable_tcp_window_scaling) 1997 opt2 |= WND_SCALE_EN(1); 1998 if (enable_ecn) { 1999 const struct tcphdr *tcph; 2000 u32 hlen = ntohl(req->hdr_len); 2001 2002 tcph = (const void *)(req + 1) + G_ETH_HDR_LEN(hlen) + 2003 G_IP_HDR_LEN(hlen); 2004 if (tcph->ece && tcph->cwr) 2005 opt2 |= CCTRL_ECN(1); 2006 } 2007 2008 rpl = cplhdr(skb); 2009 INIT_TP_WR(rpl, ep->hwtid); 2010 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 2011 ep->hwtid)); 2012 rpl->opt0 = cpu_to_be64(opt0); 2013 rpl->opt2 = cpu_to_be32(opt2); 2014 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 2015 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 2016 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 2017 2018 return; 2019 } 2020 2021 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, struct sk_buff *skb) 2022 { 2023 PDBG("%s c4iw_dev %p tid %u\n", __func__, dev, hwtid); 2024 BUG_ON(skb_cloned(skb)); 2025 skb_trim(skb, sizeof(struct cpl_tid_release)); 2026 skb_get(skb); 2027 release_tid(&dev->rdev, hwtid, skb); 2028 return; 2029 } 2030 2031 static void get_4tuple(struct cpl_pass_accept_req *req, int *iptype, 2032 __u8 *local_ip, __u8 *peer_ip, 2033 __be16 *local_port, __be16 *peer_port) 2034 { 2035 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 2036 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 2037 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 2038 struct ipv6hdr *ip6 = (struct ipv6hdr *)((u8 *)(req + 1) + eth_len); 2039 struct tcphdr *tcp = (struct tcphdr *) 2040 ((u8 *)(req + 1) + eth_len + ip_len); 2041 2042 if (ip->version == 4) { 2043 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 2044 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 2045 ntohs(tcp->dest)); 2046 *iptype = 4; 2047 memcpy(peer_ip, &ip->saddr, 4); 2048 memcpy(local_ip, &ip->daddr, 4); 2049 } else { 2050 PDBG("%s saddr %pI6 daddr %pI6 sport %u dport %u\n", __func__, 2051 ip6->saddr.s6_addr, ip6->daddr.s6_addr, ntohs(tcp->source), 2052 ntohs(tcp->dest)); 2053 *iptype = 6; 2054 memcpy(peer_ip, ip6->saddr.s6_addr, 16); 2055 memcpy(local_ip, ip6->daddr.s6_addr, 16); 2056 } 2057 *peer_port = tcp->source; 2058 *local_port = tcp->dest; 2059 2060 return; 2061 } 2062 2063 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 2064 { 2065 struct c4iw_ep *child_ep = NULL, *parent_ep; 2066 struct cpl_pass_accept_req *req = cplhdr(skb); 2067 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 2068 struct tid_info *t = dev->rdev.lldi.tids; 2069 unsigned int hwtid = GET_TID(req); 2070 struct dst_entry *dst; 2071 __u8 local_ip[16], peer_ip[16]; 2072 __be16 local_port, peer_port; 2073 int err; 2074 u16 peer_mss = ntohs(req->tcpopt.mss); 2075 int iptype; 2076 2077 parent_ep = lookup_stid(t, stid); 2078 if (!parent_ep) { 2079 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 2080 goto reject; 2081 } 2082 2083 if (state_read(&parent_ep->com) != LISTEN) { 2084 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 2085 __func__); 2086 goto reject; 2087 } 2088 2089 get_4tuple(req, &iptype, local_ip, peer_ip, &local_port, &peer_port); 2090 2091 /* Find output route */ 2092 if (iptype == 4) { 2093 PDBG("%s parent ep %p hwtid %u laddr %pI4 raddr %pI4 lport %d rport %d peer_mss %d\n" 2094 , __func__, parent_ep, hwtid, 2095 local_ip, peer_ip, ntohs(local_port), 2096 ntohs(peer_port), peer_mss); 2097 dst = find_route(dev, *(__be32 *)local_ip, *(__be32 *)peer_ip, 2098 local_port, peer_port, 2099 GET_POPEN_TOS(ntohl(req->tos_stid))); 2100 } else { 2101 PDBG("%s parent ep %p hwtid %u laddr %pI6 raddr %pI6 lport %d rport %d peer_mss %d\n" 2102 , __func__, parent_ep, hwtid, 2103 local_ip, peer_ip, ntohs(local_port), 2104 ntohs(peer_port), peer_mss); 2105 dst = find_route6(dev, local_ip, peer_ip, local_port, peer_port, 2106 PASS_OPEN_TOS(ntohl(req->tos_stid)), 2107 ((struct sockaddr_in6 *) 2108 &parent_ep->com.local_addr)->sin6_scope_id); 2109 } 2110 if (!dst) { 2111 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 2112 __func__); 2113 goto reject; 2114 } 2115 2116 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 2117 if (!child_ep) { 2118 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 2119 __func__); 2120 dst_release(dst); 2121 goto reject; 2122 } 2123 2124 err = import_ep(child_ep, iptype, peer_ip, dst, dev, false); 2125 if (err) { 2126 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 2127 __func__); 2128 dst_release(dst); 2129 kfree(child_ep); 2130 goto reject; 2131 } 2132 2133 if (peer_mss && child_ep->mtu > (peer_mss + 40)) 2134 child_ep->mtu = peer_mss + 40; 2135 2136 state_set(&child_ep->com, CONNECTING); 2137 child_ep->com.dev = dev; 2138 child_ep->com.cm_id = NULL; 2139 if (iptype == 4) { 2140 struct sockaddr_in *sin = (struct sockaddr_in *) 2141 &child_ep->com.local_addr; 2142 sin->sin_family = PF_INET; 2143 sin->sin_port = local_port; 2144 sin->sin_addr.s_addr = *(__be32 *)local_ip; 2145 sin = (struct sockaddr_in *)&child_ep->com.remote_addr; 2146 sin->sin_family = PF_INET; 2147 sin->sin_port = peer_port; 2148 sin->sin_addr.s_addr = *(__be32 *)peer_ip; 2149 } else { 2150 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) 2151 &child_ep->com.local_addr; 2152 sin6->sin6_family = PF_INET6; 2153 sin6->sin6_port = local_port; 2154 memcpy(sin6->sin6_addr.s6_addr, local_ip, 16); 2155 sin6 = (struct sockaddr_in6 *)&child_ep->com.remote_addr; 2156 sin6->sin6_family = PF_INET6; 2157 sin6->sin6_port = peer_port; 2158 memcpy(sin6->sin6_addr.s6_addr, peer_ip, 16); 2159 } 2160 c4iw_get_ep(&parent_ep->com); 2161 child_ep->parent_ep = parent_ep; 2162 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 2163 child_ep->dst = dst; 2164 child_ep->hwtid = hwtid; 2165 2166 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 2167 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 2168 2169 init_timer(&child_ep->timer); 2170 cxgb4_insert_tid(t, child_ep, hwtid); 2171 insert_handle(dev, &dev->hwtid_idr, child_ep, child_ep->hwtid); 2172 accept_cr(child_ep, skb, req); 2173 set_bit(PASS_ACCEPT_REQ, &child_ep->com.history); 2174 goto out; 2175 reject: 2176 reject_cr(dev, hwtid, skb); 2177 out: 2178 return 0; 2179 } 2180 2181 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 2182 { 2183 struct c4iw_ep *ep; 2184 struct cpl_pass_establish *req = cplhdr(skb); 2185 struct tid_info *t = dev->rdev.lldi.tids; 2186 unsigned int tid = GET_TID(req); 2187 2188 ep = lookup_tid(t, tid); 2189 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2190 ep->snd_seq = be32_to_cpu(req->snd_isn); 2191 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 2192 2193 PDBG("%s ep %p hwtid %u tcp_opt 0x%02x\n", __func__, ep, tid, 2194 ntohs(req->tcp_opt)); 2195 2196 set_emss(ep, ntohs(req->tcp_opt)); 2197 2198 dst_confirm(ep->dst); 2199 state_set(&ep->com, MPA_REQ_WAIT); 2200 start_ep_timer(ep); 2201 send_flowc(ep, skb); 2202 set_bit(PASS_ESTAB, &ep->com.history); 2203 2204 return 0; 2205 } 2206 2207 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 2208 { 2209 struct cpl_peer_close *hdr = cplhdr(skb); 2210 struct c4iw_ep *ep; 2211 struct c4iw_qp_attributes attrs; 2212 int disconnect = 1; 2213 int release = 0; 2214 struct tid_info *t = dev->rdev.lldi.tids; 2215 unsigned int tid = GET_TID(hdr); 2216 int ret; 2217 2218 ep = lookup_tid(t, tid); 2219 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2220 dst_confirm(ep->dst); 2221 2222 set_bit(PEER_CLOSE, &ep->com.history); 2223 mutex_lock(&ep->com.mutex); 2224 switch (ep->com.state) { 2225 case MPA_REQ_WAIT: 2226 __state_set(&ep->com, CLOSING); 2227 break; 2228 case MPA_REQ_SENT: 2229 __state_set(&ep->com, CLOSING); 2230 connect_reply_upcall(ep, -ECONNRESET); 2231 break; 2232 case MPA_REQ_RCVD: 2233 2234 /* 2235 * We're gonna mark this puppy DEAD, but keep 2236 * the reference on it until the ULP accepts or 2237 * rejects the CR. Also wake up anyone waiting 2238 * in rdma connection migration (see c4iw_accept_cr()). 2239 */ 2240 __state_set(&ep->com, CLOSING); 2241 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2242 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2243 break; 2244 case MPA_REP_SENT: 2245 __state_set(&ep->com, CLOSING); 2246 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 2247 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2248 break; 2249 case FPDU_MODE: 2250 start_ep_timer(ep); 2251 __state_set(&ep->com, CLOSING); 2252 attrs.next_state = C4IW_QP_STATE_CLOSING; 2253 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2254 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2255 if (ret != -ECONNRESET) { 2256 peer_close_upcall(ep); 2257 disconnect = 1; 2258 } 2259 break; 2260 case ABORTING: 2261 disconnect = 0; 2262 break; 2263 case CLOSING: 2264 __state_set(&ep->com, MORIBUND); 2265 disconnect = 0; 2266 break; 2267 case MORIBUND: 2268 stop_ep_timer(ep); 2269 if (ep->com.cm_id && ep->com.qp) { 2270 attrs.next_state = C4IW_QP_STATE_IDLE; 2271 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2272 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2273 } 2274 close_complete_upcall(ep, 0); 2275 __state_set(&ep->com, DEAD); 2276 release = 1; 2277 disconnect = 0; 2278 break; 2279 case DEAD: 2280 disconnect = 0; 2281 break; 2282 default: 2283 BUG_ON(1); 2284 } 2285 mutex_unlock(&ep->com.mutex); 2286 if (disconnect) 2287 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2288 if (release) 2289 release_ep_resources(ep); 2290 return 0; 2291 } 2292 2293 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 2294 { 2295 struct cpl_abort_req_rss *req = cplhdr(skb); 2296 struct c4iw_ep *ep; 2297 struct cpl_abort_rpl *rpl; 2298 struct sk_buff *rpl_skb; 2299 struct c4iw_qp_attributes attrs; 2300 int ret; 2301 int release = 0; 2302 struct tid_info *t = dev->rdev.lldi.tids; 2303 unsigned int tid = GET_TID(req); 2304 2305 ep = lookup_tid(t, tid); 2306 if (is_neg_adv(req->status)) { 2307 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2308 ep->hwtid); 2309 return 0; 2310 } 2311 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2312 ep->com.state); 2313 set_bit(PEER_ABORT, &ep->com.history); 2314 2315 /* 2316 * Wake up any threads in rdma_init() or rdma_fini(). 2317 * However, this is not needed if com state is just 2318 * MPA_REQ_SENT 2319 */ 2320 if (ep->com.state != MPA_REQ_SENT) 2321 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2322 2323 mutex_lock(&ep->com.mutex); 2324 switch (ep->com.state) { 2325 case CONNECTING: 2326 break; 2327 case MPA_REQ_WAIT: 2328 stop_ep_timer(ep); 2329 break; 2330 case MPA_REQ_SENT: 2331 stop_ep_timer(ep); 2332 if (mpa_rev == 1 || (mpa_rev == 2 && ep->tried_with_mpa_v1)) 2333 connect_reply_upcall(ep, -ECONNRESET); 2334 else { 2335 /* 2336 * we just don't send notification upwards because we 2337 * want to retry with mpa_v1 without upper layers even 2338 * knowing it. 2339 * 2340 * do some housekeeping so as to re-initiate the 2341 * connection 2342 */ 2343 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 2344 mpa_rev); 2345 ep->retry_with_mpa_v1 = 1; 2346 } 2347 break; 2348 case MPA_REP_SENT: 2349 break; 2350 case MPA_REQ_RCVD: 2351 break; 2352 case MORIBUND: 2353 case CLOSING: 2354 stop_ep_timer(ep); 2355 /*FALLTHROUGH*/ 2356 case FPDU_MODE: 2357 if (ep->com.cm_id && ep->com.qp) { 2358 attrs.next_state = C4IW_QP_STATE_ERROR; 2359 ret = c4iw_modify_qp(ep->com.qp->rhp, 2360 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2361 &attrs, 1); 2362 if (ret) 2363 printk(KERN_ERR MOD 2364 "%s - qp <- error failed!\n", 2365 __func__); 2366 } 2367 peer_abort_upcall(ep); 2368 break; 2369 case ABORTING: 2370 break; 2371 case DEAD: 2372 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 2373 mutex_unlock(&ep->com.mutex); 2374 return 0; 2375 default: 2376 BUG_ON(1); 2377 break; 2378 } 2379 dst_confirm(ep->dst); 2380 if (ep->com.state != ABORTING) { 2381 __state_set(&ep->com, DEAD); 2382 /* we don't release if we want to retry with mpa_v1 */ 2383 if (!ep->retry_with_mpa_v1) 2384 release = 1; 2385 } 2386 mutex_unlock(&ep->com.mutex); 2387 2388 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 2389 if (!rpl_skb) { 2390 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 2391 __func__); 2392 release = 1; 2393 goto out; 2394 } 2395 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 2396 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 2397 INIT_TP_WR(rpl, ep->hwtid); 2398 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 2399 rpl->cmd = CPL_ABORT_NO_RST; 2400 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 2401 out: 2402 if (release) 2403 release_ep_resources(ep); 2404 else if (ep->retry_with_mpa_v1) { 2405 remove_handle(ep->com.dev, &ep->com.dev->hwtid_idr, ep->hwtid); 2406 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 2407 dst_release(ep->dst); 2408 cxgb4_l2t_release(ep->l2t); 2409 c4iw_reconnect(ep); 2410 } 2411 2412 return 0; 2413 } 2414 2415 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2416 { 2417 struct c4iw_ep *ep; 2418 struct c4iw_qp_attributes attrs; 2419 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2420 int release = 0; 2421 struct tid_info *t = dev->rdev.lldi.tids; 2422 unsigned int tid = GET_TID(rpl); 2423 2424 ep = lookup_tid(t, tid); 2425 2426 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2427 BUG_ON(!ep); 2428 2429 /* The cm_id may be null if we failed to connect */ 2430 mutex_lock(&ep->com.mutex); 2431 switch (ep->com.state) { 2432 case CLOSING: 2433 __state_set(&ep->com, MORIBUND); 2434 break; 2435 case MORIBUND: 2436 stop_ep_timer(ep); 2437 if ((ep->com.cm_id) && (ep->com.qp)) { 2438 attrs.next_state = C4IW_QP_STATE_IDLE; 2439 c4iw_modify_qp(ep->com.qp->rhp, 2440 ep->com.qp, 2441 C4IW_QP_ATTR_NEXT_STATE, 2442 &attrs, 1); 2443 } 2444 close_complete_upcall(ep, 0); 2445 __state_set(&ep->com, DEAD); 2446 release = 1; 2447 break; 2448 case ABORTING: 2449 case DEAD: 2450 break; 2451 default: 2452 BUG_ON(1); 2453 break; 2454 } 2455 mutex_unlock(&ep->com.mutex); 2456 if (release) 2457 release_ep_resources(ep); 2458 return 0; 2459 } 2460 2461 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2462 { 2463 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2464 struct tid_info *t = dev->rdev.lldi.tids; 2465 unsigned int tid = GET_TID(rpl); 2466 struct c4iw_ep *ep; 2467 struct c4iw_qp_attributes attrs; 2468 2469 ep = lookup_tid(t, tid); 2470 BUG_ON(!ep); 2471 2472 if (ep && ep->com.qp) { 2473 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2474 ep->com.qp->wq.sq.qid); 2475 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2476 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2477 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2478 } else 2479 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2480 2481 return 0; 2482 } 2483 2484 /* 2485 * Upcall from the adapter indicating data has been transmitted. 2486 * For us its just the single MPA request or reply. We can now free 2487 * the skb holding the mpa message. 2488 */ 2489 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2490 { 2491 struct c4iw_ep *ep; 2492 struct cpl_fw4_ack *hdr = cplhdr(skb); 2493 u8 credits = hdr->credits; 2494 unsigned int tid = GET_TID(hdr); 2495 struct tid_info *t = dev->rdev.lldi.tids; 2496 2497 2498 ep = lookup_tid(t, tid); 2499 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2500 if (credits == 0) { 2501 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2502 __func__, ep, ep->hwtid, state_read(&ep->com)); 2503 return 0; 2504 } 2505 2506 dst_confirm(ep->dst); 2507 if (ep->mpa_skb) { 2508 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2509 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2510 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2511 kfree_skb(ep->mpa_skb); 2512 ep->mpa_skb = NULL; 2513 } 2514 return 0; 2515 } 2516 2517 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2518 { 2519 int err = 0; 2520 int disconnect = 0; 2521 struct c4iw_ep *ep = to_ep(cm_id); 2522 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2523 2524 mutex_lock(&ep->com.mutex); 2525 if (ep->com.state == DEAD) { 2526 mutex_unlock(&ep->com.mutex); 2527 c4iw_put_ep(&ep->com); 2528 return -ECONNRESET; 2529 } 2530 set_bit(ULP_REJECT, &ep->com.history); 2531 BUG_ON(ep->com.state != MPA_REQ_RCVD); 2532 if (mpa_rev == 0) 2533 abort_connection(ep, NULL, GFP_KERNEL); 2534 else { 2535 err = send_mpa_reject(ep, pdata, pdata_len); 2536 disconnect = 1; 2537 } 2538 mutex_unlock(&ep->com.mutex); 2539 if (disconnect) 2540 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2541 c4iw_put_ep(&ep->com); 2542 return 0; 2543 } 2544 2545 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2546 { 2547 int err; 2548 struct c4iw_qp_attributes attrs; 2549 enum c4iw_qp_attr_mask mask; 2550 struct c4iw_ep *ep = to_ep(cm_id); 2551 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2552 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2553 2554 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2555 2556 mutex_lock(&ep->com.mutex); 2557 if (ep->com.state == DEAD) { 2558 err = -ECONNRESET; 2559 goto err; 2560 } 2561 2562 BUG_ON(ep->com.state != MPA_REQ_RCVD); 2563 BUG_ON(!qp); 2564 2565 set_bit(ULP_ACCEPT, &ep->com.history); 2566 if ((conn_param->ord > c4iw_max_read_depth) || 2567 (conn_param->ird > c4iw_max_read_depth)) { 2568 abort_connection(ep, NULL, GFP_KERNEL); 2569 err = -EINVAL; 2570 goto err; 2571 } 2572 2573 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2574 if (conn_param->ord > ep->ird) { 2575 ep->ird = conn_param->ird; 2576 ep->ord = conn_param->ord; 2577 send_mpa_reject(ep, conn_param->private_data, 2578 conn_param->private_data_len); 2579 abort_connection(ep, NULL, GFP_KERNEL); 2580 err = -ENOMEM; 2581 goto err; 2582 } 2583 if (conn_param->ird > ep->ord) { 2584 if (!ep->ord) 2585 conn_param->ird = 1; 2586 else { 2587 abort_connection(ep, NULL, GFP_KERNEL); 2588 err = -ENOMEM; 2589 goto err; 2590 } 2591 } 2592 2593 } 2594 ep->ird = conn_param->ird; 2595 ep->ord = conn_param->ord; 2596 2597 if (ep->mpa_attr.version != 2) 2598 if (peer2peer && ep->ird == 0) 2599 ep->ird = 1; 2600 2601 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2602 2603 cm_id->add_ref(cm_id); 2604 ep->com.cm_id = cm_id; 2605 ep->com.qp = qp; 2606 ref_qp(ep); 2607 2608 /* bind QP to EP and move to RTS */ 2609 attrs.mpa_attr = ep->mpa_attr; 2610 attrs.max_ird = ep->ird; 2611 attrs.max_ord = ep->ord; 2612 attrs.llp_stream_handle = ep; 2613 attrs.next_state = C4IW_QP_STATE_RTS; 2614 2615 /* bind QP and TID with INIT_WR */ 2616 mask = C4IW_QP_ATTR_NEXT_STATE | 2617 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2618 C4IW_QP_ATTR_MPA_ATTR | 2619 C4IW_QP_ATTR_MAX_IRD | 2620 C4IW_QP_ATTR_MAX_ORD; 2621 2622 err = c4iw_modify_qp(ep->com.qp->rhp, 2623 ep->com.qp, mask, &attrs, 1); 2624 if (err) 2625 goto err1; 2626 err = send_mpa_reply(ep, conn_param->private_data, 2627 conn_param->private_data_len); 2628 if (err) 2629 goto err1; 2630 2631 __state_set(&ep->com, FPDU_MODE); 2632 established_upcall(ep); 2633 mutex_unlock(&ep->com.mutex); 2634 c4iw_put_ep(&ep->com); 2635 return 0; 2636 err1: 2637 ep->com.cm_id = NULL; 2638 cm_id->rem_ref(cm_id); 2639 err: 2640 mutex_unlock(&ep->com.mutex); 2641 c4iw_put_ep(&ep->com); 2642 return err; 2643 } 2644 2645 static int pick_local_ipaddrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2646 { 2647 struct in_device *ind; 2648 int found = 0; 2649 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2650 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2651 2652 ind = in_dev_get(dev->rdev.lldi.ports[0]); 2653 if (!ind) 2654 return -EADDRNOTAVAIL; 2655 for_primary_ifa(ind) { 2656 laddr->sin_addr.s_addr = ifa->ifa_address; 2657 raddr->sin_addr.s_addr = ifa->ifa_address; 2658 found = 1; 2659 break; 2660 } 2661 endfor_ifa(ind); 2662 in_dev_put(ind); 2663 return found ? 0 : -EADDRNOTAVAIL; 2664 } 2665 2666 static int get_lladdr(struct net_device *dev, struct in6_addr *addr, 2667 unsigned char banned_flags) 2668 { 2669 struct inet6_dev *idev; 2670 int err = -EADDRNOTAVAIL; 2671 2672 rcu_read_lock(); 2673 idev = __in6_dev_get(dev); 2674 if (idev != NULL) { 2675 struct inet6_ifaddr *ifp; 2676 2677 read_lock_bh(&idev->lock); 2678 list_for_each_entry(ifp, &idev->addr_list, if_list) { 2679 if (ifp->scope == IFA_LINK && 2680 !(ifp->flags & banned_flags)) { 2681 memcpy(addr, &ifp->addr, 16); 2682 err = 0; 2683 break; 2684 } 2685 } 2686 read_unlock_bh(&idev->lock); 2687 } 2688 rcu_read_unlock(); 2689 return err; 2690 } 2691 2692 static int pick_local_ip6addrs(struct c4iw_dev *dev, struct iw_cm_id *cm_id) 2693 { 2694 struct in6_addr uninitialized_var(addr); 2695 struct sockaddr_in6 *la6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2696 struct sockaddr_in6 *ra6 = (struct sockaddr_in6 *)&cm_id->remote_addr; 2697 2698 if (get_lladdr(dev->rdev.lldi.ports[0], &addr, IFA_F_TENTATIVE)) { 2699 memcpy(la6->sin6_addr.s6_addr, &addr, 16); 2700 memcpy(ra6->sin6_addr.s6_addr, &addr, 16); 2701 return 0; 2702 } 2703 return -EADDRNOTAVAIL; 2704 } 2705 2706 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2707 { 2708 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2709 struct c4iw_ep *ep; 2710 int err = 0; 2711 struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->local_addr; 2712 struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->remote_addr; 2713 struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&cm_id->local_addr; 2714 struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *) 2715 &cm_id->remote_addr; 2716 __u8 *ra; 2717 int iptype; 2718 2719 if ((conn_param->ord > c4iw_max_read_depth) || 2720 (conn_param->ird > c4iw_max_read_depth)) { 2721 err = -EINVAL; 2722 goto out; 2723 } 2724 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2725 if (!ep) { 2726 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2727 err = -ENOMEM; 2728 goto out; 2729 } 2730 init_timer(&ep->timer); 2731 ep->plen = conn_param->private_data_len; 2732 if (ep->plen) 2733 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2734 conn_param->private_data, ep->plen); 2735 ep->ird = conn_param->ird; 2736 ep->ord = conn_param->ord; 2737 2738 if (peer2peer && ep->ord == 0) 2739 ep->ord = 1; 2740 2741 cm_id->add_ref(cm_id); 2742 ep->com.dev = dev; 2743 ep->com.cm_id = cm_id; 2744 ep->com.qp = get_qhp(dev, conn_param->qpn); 2745 if (!ep->com.qp) { 2746 PDBG("%s qpn 0x%x not found!\n", __func__, conn_param->qpn); 2747 err = -EINVAL; 2748 goto fail2; 2749 } 2750 ref_qp(ep); 2751 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2752 ep->com.qp, cm_id); 2753 2754 /* 2755 * Allocate an active TID to initiate a TCP connection. 2756 */ 2757 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2758 if (ep->atid == -1) { 2759 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2760 err = -ENOMEM; 2761 goto fail2; 2762 } 2763 insert_handle(dev, &dev->atid_idr, ep, ep->atid); 2764 2765 if (cm_id->remote_addr.ss_family == AF_INET) { 2766 iptype = 4; 2767 ra = (__u8 *)&raddr->sin_addr; 2768 2769 /* 2770 * Handle loopback requests to INADDR_ANY. 2771 */ 2772 if ((__force int)raddr->sin_addr.s_addr == INADDR_ANY) { 2773 err = pick_local_ipaddrs(dev, cm_id); 2774 if (err) 2775 goto fail2; 2776 } 2777 2778 /* find a route */ 2779 PDBG("%s saddr %pI4 sport 0x%x raddr %pI4 rport 0x%x\n", 2780 __func__, &laddr->sin_addr, ntohs(laddr->sin_port), 2781 ra, ntohs(raddr->sin_port)); 2782 ep->dst = find_route(dev, laddr->sin_addr.s_addr, 2783 raddr->sin_addr.s_addr, laddr->sin_port, 2784 raddr->sin_port, 0); 2785 } else { 2786 iptype = 6; 2787 ra = (__u8 *)&raddr6->sin6_addr; 2788 2789 /* 2790 * Handle loopback requests to INADDR_ANY. 2791 */ 2792 if (ipv6_addr_type(&raddr6->sin6_addr) == IPV6_ADDR_ANY) { 2793 err = pick_local_ip6addrs(dev, cm_id); 2794 if (err) 2795 goto fail2; 2796 } 2797 2798 /* find a route */ 2799 PDBG("%s saddr %pI6 sport 0x%x raddr %pI6 rport 0x%x\n", 2800 __func__, laddr6->sin6_addr.s6_addr, 2801 ntohs(laddr6->sin6_port), 2802 raddr6->sin6_addr.s6_addr, ntohs(raddr6->sin6_port)); 2803 ep->dst = find_route6(dev, laddr6->sin6_addr.s6_addr, 2804 raddr6->sin6_addr.s6_addr, 2805 laddr6->sin6_port, raddr6->sin6_port, 0, 2806 raddr6->sin6_scope_id); 2807 } 2808 if (!ep->dst) { 2809 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2810 err = -EHOSTUNREACH; 2811 goto fail3; 2812 } 2813 2814 err = import_ep(ep, iptype, ra, ep->dst, ep->com.dev, true); 2815 if (err) { 2816 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2817 goto fail4; 2818 } 2819 2820 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2821 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2822 ep->l2t->idx); 2823 2824 state_set(&ep->com, CONNECTING); 2825 ep->tos = 0; 2826 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2827 sizeof(ep->com.local_addr)); 2828 memcpy(&ep->com.remote_addr, &cm_id->remote_addr, 2829 sizeof(ep->com.remote_addr)); 2830 2831 /* send connect request to rnic */ 2832 err = send_connect(ep); 2833 if (!err) 2834 goto out; 2835 2836 cxgb4_l2t_release(ep->l2t); 2837 fail4: 2838 dst_release(ep->dst); 2839 fail3: 2840 remove_handle(ep->com.dev, &ep->com.dev->atid_idr, ep->atid); 2841 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2842 fail2: 2843 cm_id->rem_ref(cm_id); 2844 c4iw_put_ep(&ep->com); 2845 out: 2846 return err; 2847 } 2848 2849 static int create_server6(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2850 { 2851 int err; 2852 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ep->com.local_addr; 2853 2854 c4iw_init_wr_wait(&ep->com.wr_wait); 2855 err = cxgb4_create_server6(ep->com.dev->rdev.lldi.ports[0], 2856 ep->stid, &sin6->sin6_addr, 2857 sin6->sin6_port, 2858 ep->com.dev->rdev.lldi.rxq_ids[0]); 2859 if (!err) 2860 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2861 &ep->com.wr_wait, 2862 0, 0, __func__); 2863 if (err) 2864 pr_err("cxgb4_create_server6/filter failed err %d stid %d laddr %pI6 lport %d\n", 2865 err, ep->stid, 2866 sin6->sin6_addr.s6_addr, ntohs(sin6->sin6_port)); 2867 return err; 2868 } 2869 2870 static int create_server4(struct c4iw_dev *dev, struct c4iw_listen_ep *ep) 2871 { 2872 int err; 2873 struct sockaddr_in *sin = (struct sockaddr_in *)&ep->com.local_addr; 2874 2875 if (dev->rdev.lldi.enable_fw_ofld_conn) { 2876 do { 2877 err = cxgb4_create_server_filter( 2878 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2879 sin->sin_addr.s_addr, sin->sin_port, 0, 2880 ep->com.dev->rdev.lldi.rxq_ids[0], 0, 0); 2881 if (err == -EBUSY) { 2882 set_current_state(TASK_UNINTERRUPTIBLE); 2883 schedule_timeout(usecs_to_jiffies(100)); 2884 } 2885 } while (err == -EBUSY); 2886 } else { 2887 c4iw_init_wr_wait(&ep->com.wr_wait); 2888 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], 2889 ep->stid, sin->sin_addr.s_addr, sin->sin_port, 2890 0, ep->com.dev->rdev.lldi.rxq_ids[0]); 2891 if (!err) 2892 err = c4iw_wait_for_reply(&ep->com.dev->rdev, 2893 &ep->com.wr_wait, 2894 0, 0, __func__); 2895 } 2896 if (err) 2897 pr_err("cxgb4_create_server/filter failed err %d stid %d laddr %pI4 lport %d\n" 2898 , err, ep->stid, 2899 &sin->sin_addr, ntohs(sin->sin_port)); 2900 return err; 2901 } 2902 2903 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2904 { 2905 int err = 0; 2906 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2907 struct c4iw_listen_ep *ep; 2908 2909 might_sleep(); 2910 2911 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2912 if (!ep) { 2913 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2914 err = -ENOMEM; 2915 goto fail1; 2916 } 2917 PDBG("%s ep %p\n", __func__, ep); 2918 cm_id->add_ref(cm_id); 2919 ep->com.cm_id = cm_id; 2920 ep->com.dev = dev; 2921 ep->backlog = backlog; 2922 memcpy(&ep->com.local_addr, &cm_id->local_addr, 2923 sizeof(ep->com.local_addr)); 2924 2925 /* 2926 * Allocate a server TID. 2927 */ 2928 if (dev->rdev.lldi.enable_fw_ofld_conn && 2929 ep->com.local_addr.ss_family == AF_INET) 2930 ep->stid = cxgb4_alloc_sftid(dev->rdev.lldi.tids, 2931 cm_id->local_addr.ss_family, ep); 2932 else 2933 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, 2934 cm_id->local_addr.ss_family, ep); 2935 2936 if (ep->stid == -1) { 2937 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2938 err = -ENOMEM; 2939 goto fail2; 2940 } 2941 insert_handle(dev, &dev->stid_idr, ep, ep->stid); 2942 state_set(&ep->com, LISTEN); 2943 if (ep->com.local_addr.ss_family == AF_INET) 2944 err = create_server4(dev, ep); 2945 else 2946 err = create_server6(dev, ep); 2947 if (!err) { 2948 cm_id->provider_data = ep; 2949 goto out; 2950 } 2951 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 2952 ep->com.local_addr.ss_family); 2953 fail2: 2954 cm_id->rem_ref(cm_id); 2955 c4iw_put_ep(&ep->com); 2956 fail1: 2957 out: 2958 return err; 2959 } 2960 2961 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2962 { 2963 int err; 2964 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2965 2966 PDBG("%s ep %p\n", __func__, ep); 2967 2968 might_sleep(); 2969 state_set(&ep->com, DEAD); 2970 if (ep->com.dev->rdev.lldi.enable_fw_ofld_conn && 2971 ep->com.local_addr.ss_family == AF_INET) { 2972 err = cxgb4_remove_server_filter( 2973 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2974 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2975 } else { 2976 c4iw_init_wr_wait(&ep->com.wr_wait); 2977 err = cxgb4_remove_server( 2978 ep->com.dev->rdev.lldi.ports[0], ep->stid, 2979 ep->com.dev->rdev.lldi.rxq_ids[0], 0); 2980 if (err) 2981 goto done; 2982 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 2983 0, 0, __func__); 2984 } 2985 remove_handle(ep->com.dev, &ep->com.dev->stid_idr, ep->stid); 2986 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, 2987 ep->com.local_addr.ss_family); 2988 done: 2989 cm_id->rem_ref(cm_id); 2990 c4iw_put_ep(&ep->com); 2991 return err; 2992 } 2993 2994 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2995 { 2996 int ret = 0; 2997 int close = 0; 2998 int fatal = 0; 2999 struct c4iw_rdev *rdev; 3000 3001 mutex_lock(&ep->com.mutex); 3002 3003 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 3004 states[ep->com.state], abrupt); 3005 3006 rdev = &ep->com.dev->rdev; 3007 if (c4iw_fatal_error(rdev)) { 3008 fatal = 1; 3009 close_complete_upcall(ep, -EIO); 3010 ep->com.state = DEAD; 3011 } 3012 switch (ep->com.state) { 3013 case MPA_REQ_WAIT: 3014 case MPA_REQ_SENT: 3015 case MPA_REQ_RCVD: 3016 case MPA_REP_SENT: 3017 case FPDU_MODE: 3018 close = 1; 3019 if (abrupt) 3020 ep->com.state = ABORTING; 3021 else { 3022 ep->com.state = CLOSING; 3023 start_ep_timer(ep); 3024 } 3025 set_bit(CLOSE_SENT, &ep->com.flags); 3026 break; 3027 case CLOSING: 3028 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 3029 close = 1; 3030 if (abrupt) { 3031 stop_ep_timer(ep); 3032 ep->com.state = ABORTING; 3033 } else 3034 ep->com.state = MORIBUND; 3035 } 3036 break; 3037 case MORIBUND: 3038 case ABORTING: 3039 case DEAD: 3040 PDBG("%s ignoring disconnect ep %p state %u\n", 3041 __func__, ep, ep->com.state); 3042 break; 3043 default: 3044 BUG(); 3045 break; 3046 } 3047 3048 if (close) { 3049 if (abrupt) { 3050 set_bit(EP_DISC_ABORT, &ep->com.history); 3051 close_complete_upcall(ep, -ECONNRESET); 3052 ret = send_abort(ep, NULL, gfp); 3053 } else { 3054 set_bit(EP_DISC_CLOSE, &ep->com.history); 3055 ret = send_halfclose(ep, gfp); 3056 } 3057 if (ret) 3058 fatal = 1; 3059 } 3060 mutex_unlock(&ep->com.mutex); 3061 if (fatal) 3062 release_ep_resources(ep); 3063 return ret; 3064 } 3065 3066 static void active_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3067 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3068 { 3069 struct c4iw_ep *ep; 3070 int atid = be32_to_cpu(req->tid); 3071 3072 ep = (struct c4iw_ep *)lookup_atid(dev->rdev.lldi.tids, 3073 (__force u32) req->tid); 3074 if (!ep) 3075 return; 3076 3077 switch (req->retval) { 3078 case FW_ENOMEM: 3079 set_bit(ACT_RETRY_NOMEM, &ep->com.history); 3080 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3081 send_fw_act_open_req(ep, atid); 3082 return; 3083 } 3084 case FW_EADDRINUSE: 3085 set_bit(ACT_RETRY_INUSE, &ep->com.history); 3086 if (ep->retry_count++ < ACT_OPEN_RETRY_COUNT) { 3087 send_fw_act_open_req(ep, atid); 3088 return; 3089 } 3090 break; 3091 default: 3092 pr_info("%s unexpected ofld conn wr retval %d\n", 3093 __func__, req->retval); 3094 break; 3095 } 3096 pr_err("active ofld_connect_wr failure %d atid %d\n", 3097 req->retval, atid); 3098 mutex_lock(&dev->rdev.stats.lock); 3099 dev->rdev.stats.act_ofld_conn_fails++; 3100 mutex_unlock(&dev->rdev.stats.lock); 3101 connect_reply_upcall(ep, status2errno(req->retval)); 3102 state_set(&ep->com, DEAD); 3103 remove_handle(dev, &dev->atid_idr, atid); 3104 cxgb4_free_atid(dev->rdev.lldi.tids, atid); 3105 dst_release(ep->dst); 3106 cxgb4_l2t_release(ep->l2t); 3107 c4iw_put_ep(&ep->com); 3108 } 3109 3110 static void passive_ofld_conn_reply(struct c4iw_dev *dev, struct sk_buff *skb, 3111 struct cpl_fw6_msg_ofld_connection_wr_rpl *req) 3112 { 3113 struct sk_buff *rpl_skb; 3114 struct cpl_pass_accept_req *cpl; 3115 int ret; 3116 3117 rpl_skb = (struct sk_buff *)(unsigned long)req->cookie; 3118 BUG_ON(!rpl_skb); 3119 if (req->retval) { 3120 PDBG("%s passive open failure %d\n", __func__, req->retval); 3121 mutex_lock(&dev->rdev.stats.lock); 3122 dev->rdev.stats.pas_ofld_conn_fails++; 3123 mutex_unlock(&dev->rdev.stats.lock); 3124 kfree_skb(rpl_skb); 3125 } else { 3126 cpl = (struct cpl_pass_accept_req *)cplhdr(rpl_skb); 3127 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 3128 (__force u32) htonl( 3129 (__force u32) req->tid))); 3130 ret = pass_accept_req(dev, rpl_skb); 3131 if (!ret) 3132 kfree_skb(rpl_skb); 3133 } 3134 return; 3135 } 3136 3137 static int deferred_fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3138 { 3139 struct cpl_fw6_msg *rpl = cplhdr(skb); 3140 struct cpl_fw6_msg_ofld_connection_wr_rpl *req; 3141 3142 switch (rpl->type) { 3143 case FW6_TYPE_CQE: 3144 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 3145 break; 3146 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3147 req = (struct cpl_fw6_msg_ofld_connection_wr_rpl *)rpl->data; 3148 switch (req->t_state) { 3149 case TCP_SYN_SENT: 3150 active_ofld_conn_reply(dev, skb, req); 3151 break; 3152 case TCP_SYN_RECV: 3153 passive_ofld_conn_reply(dev, skb, req); 3154 break; 3155 default: 3156 pr_err("%s unexpected ofld conn wr state %d\n", 3157 __func__, req->t_state); 3158 break; 3159 } 3160 break; 3161 } 3162 return 0; 3163 } 3164 3165 static void build_cpl_pass_accept_req(struct sk_buff *skb, int stid , u8 tos) 3166 { 3167 u32 l2info; 3168 u16 vlantag, len, hdr_len, eth_hdr_len; 3169 u8 intf; 3170 struct cpl_rx_pkt *cpl = cplhdr(skb); 3171 struct cpl_pass_accept_req *req; 3172 struct tcp_options_received tmp_opt; 3173 struct c4iw_dev *dev; 3174 3175 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3176 /* Store values from cpl_rx_pkt in temporary location. */ 3177 vlantag = (__force u16) cpl->vlan; 3178 len = (__force u16) cpl->len; 3179 l2info = (__force u32) cpl->l2info; 3180 hdr_len = (__force u16) cpl->hdr_len; 3181 intf = cpl->iff; 3182 3183 __skb_pull(skb, sizeof(*req) + sizeof(struct rss_header)); 3184 3185 /* 3186 * We need to parse the TCP options from SYN packet. 3187 * to generate cpl_pass_accept_req. 3188 */ 3189 memset(&tmp_opt, 0, sizeof(tmp_opt)); 3190 tcp_clear_options(&tmp_opt); 3191 tcp_parse_options(skb, &tmp_opt, 0, NULL); 3192 3193 req = (struct cpl_pass_accept_req *)__skb_push(skb, sizeof(*req)); 3194 memset(req, 0, sizeof(*req)); 3195 req->l2info = cpu_to_be16(V_SYN_INTF(intf) | 3196 V_SYN_MAC_IDX(G_RX_MACIDX( 3197 (__force int) htonl(l2info))) | 3198 F_SYN_XACT_MATCH); 3199 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3200 G_RX_ETHHDR_LEN((__force int) htonl(l2info)) : 3201 G_RX_T5_ETHHDR_LEN((__force int) htonl(l2info)); 3202 req->hdr_len = cpu_to_be32(V_SYN_RX_CHAN(G_RX_CHAN( 3203 (__force int) htonl(l2info))) | 3204 V_TCP_HDR_LEN(G_RX_TCPHDR_LEN( 3205 (__force int) htons(hdr_len))) | 3206 V_IP_HDR_LEN(G_RX_IPHDR_LEN( 3207 (__force int) htons(hdr_len))) | 3208 V_ETH_HDR_LEN(G_RX_ETHHDR_LEN(eth_hdr_len))); 3209 req->vlan = (__force __be16) vlantag; 3210 req->len = (__force __be16) len; 3211 req->tos_stid = cpu_to_be32(PASS_OPEN_TID(stid) | 3212 PASS_OPEN_TOS(tos)); 3213 req->tcpopt.mss = htons(tmp_opt.mss_clamp); 3214 if (tmp_opt.wscale_ok) 3215 req->tcpopt.wsf = tmp_opt.snd_wscale; 3216 req->tcpopt.tstamp = tmp_opt.saw_tstamp; 3217 if (tmp_opt.sack_ok) 3218 req->tcpopt.sack = 1; 3219 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_REQ, 0)); 3220 return; 3221 } 3222 3223 static void send_fw_pass_open_req(struct c4iw_dev *dev, struct sk_buff *skb, 3224 __be32 laddr, __be16 lport, 3225 __be32 raddr, __be16 rport, 3226 u32 rcv_isn, u32 filter, u16 window, 3227 u32 rss_qid, u8 port_id) 3228 { 3229 struct sk_buff *req_skb; 3230 struct fw_ofld_connection_wr *req; 3231 struct cpl_pass_accept_req *cpl = cplhdr(skb); 3232 int ret; 3233 3234 req_skb = alloc_skb(sizeof(struct fw_ofld_connection_wr), GFP_KERNEL); 3235 req = (struct fw_ofld_connection_wr *)__skb_put(req_skb, sizeof(*req)); 3236 memset(req, 0, sizeof(*req)); 3237 req->op_compl = htonl(V_WR_OP(FW_OFLD_CONNECTION_WR) | FW_WR_COMPL(1)); 3238 req->len16_pkd = htonl(FW_WR_LEN16(DIV_ROUND_UP(sizeof(*req), 16))); 3239 req->le.version_cpl = htonl(F_FW_OFLD_CONNECTION_WR_CPL); 3240 req->le.filter = (__force __be32) filter; 3241 req->le.lport = lport; 3242 req->le.pport = rport; 3243 req->le.u.ipv4.lip = laddr; 3244 req->le.u.ipv4.pip = raddr; 3245 req->tcb.rcv_nxt = htonl(rcv_isn + 1); 3246 req->tcb.rcv_adv = htons(window); 3247 req->tcb.t_state_to_astid = 3248 htonl(V_FW_OFLD_CONNECTION_WR_T_STATE(TCP_SYN_RECV) | 3249 V_FW_OFLD_CONNECTION_WR_RCV_SCALE(cpl->tcpopt.wsf) | 3250 V_FW_OFLD_CONNECTION_WR_ASTID( 3251 GET_PASS_OPEN_TID(ntohl(cpl->tos_stid)))); 3252 3253 /* 3254 * We store the qid in opt2 which will be used by the firmware 3255 * to send us the wr response. 3256 */ 3257 req->tcb.opt2 = htonl(V_RSS_QUEUE(rss_qid)); 3258 3259 /* 3260 * We initialize the MSS index in TCB to 0xF. 3261 * So that when driver sends cpl_pass_accept_rpl 3262 * TCB picks up the correct value. If this was 0 3263 * TP will ignore any value > 0 for MSS index. 3264 */ 3265 req->tcb.opt0 = cpu_to_be64(V_MSS_IDX(0xF)); 3266 req->cookie = (unsigned long)skb; 3267 3268 set_wr_txq(req_skb, CPL_PRIORITY_CONTROL, port_id); 3269 ret = cxgb4_ofld_send(dev->rdev.lldi.ports[0], req_skb); 3270 if (ret < 0) { 3271 pr_err("%s - cxgb4_ofld_send error %d - dropping\n", __func__, 3272 ret); 3273 kfree_skb(skb); 3274 kfree_skb(req_skb); 3275 } 3276 } 3277 3278 /* 3279 * Handler for CPL_RX_PKT message. Need to handle cpl_rx_pkt 3280 * messages when a filter is being used instead of server to 3281 * redirect a syn packet. When packets hit filter they are redirected 3282 * to the offload queue and driver tries to establish the connection 3283 * using firmware work request. 3284 */ 3285 static int rx_pkt(struct c4iw_dev *dev, struct sk_buff *skb) 3286 { 3287 int stid; 3288 unsigned int filter; 3289 struct ethhdr *eh = NULL; 3290 struct vlan_ethhdr *vlan_eh = NULL; 3291 struct iphdr *iph; 3292 struct tcphdr *tcph; 3293 struct rss_header *rss = (void *)skb->data; 3294 struct cpl_rx_pkt *cpl = (void *)skb->data; 3295 struct cpl_pass_accept_req *req = (void *)(rss + 1); 3296 struct l2t_entry *e; 3297 struct dst_entry *dst; 3298 struct c4iw_ep *lep; 3299 u16 window; 3300 struct port_info *pi; 3301 struct net_device *pdev; 3302 u16 rss_qid, eth_hdr_len; 3303 int step; 3304 u32 tx_chan; 3305 struct neighbour *neigh; 3306 3307 /* Drop all non-SYN packets */ 3308 if (!(cpl->l2info & cpu_to_be32(F_RXF_SYN))) 3309 goto reject; 3310 3311 /* 3312 * Drop all packets which did not hit the filter. 3313 * Unlikely to happen. 3314 */ 3315 if (!(rss->filter_hit && rss->filter_tid)) 3316 goto reject; 3317 3318 /* 3319 * Calculate the server tid from filter hit index from cpl_rx_pkt. 3320 */ 3321 stid = (__force int) cpu_to_be32((__force u32) rss->hash_val); 3322 3323 lep = (struct c4iw_ep *)lookup_stid(dev->rdev.lldi.tids, stid); 3324 if (!lep) { 3325 PDBG("%s connect request on invalid stid %d\n", __func__, stid); 3326 goto reject; 3327 } 3328 3329 eth_hdr_len = is_t4(dev->rdev.lldi.adapter_type) ? 3330 G_RX_ETHHDR_LEN(htonl(cpl->l2info)) : 3331 G_RX_T5_ETHHDR_LEN(htonl(cpl->l2info)); 3332 if (eth_hdr_len == ETH_HLEN) { 3333 eh = (struct ethhdr *)(req + 1); 3334 iph = (struct iphdr *)(eh + 1); 3335 } else { 3336 vlan_eh = (struct vlan_ethhdr *)(req + 1); 3337 iph = (struct iphdr *)(vlan_eh + 1); 3338 skb->vlan_tci = ntohs(cpl->vlan); 3339 } 3340 3341 if (iph->version != 0x4) 3342 goto reject; 3343 3344 tcph = (struct tcphdr *)(iph + 1); 3345 skb_set_network_header(skb, (void *)iph - (void *)rss); 3346 skb_set_transport_header(skb, (void *)tcph - (void *)rss); 3347 skb_get(skb); 3348 3349 PDBG("%s lip 0x%x lport %u pip 0x%x pport %u tos %d\n", __func__, 3350 ntohl(iph->daddr), ntohs(tcph->dest), ntohl(iph->saddr), 3351 ntohs(tcph->source), iph->tos); 3352 3353 dst = find_route(dev, iph->daddr, iph->saddr, tcph->dest, tcph->source, 3354 iph->tos); 3355 if (!dst) { 3356 pr_err("%s - failed to find dst entry!\n", 3357 __func__); 3358 goto reject; 3359 } 3360 neigh = dst_neigh_lookup_skb(dst, skb); 3361 3362 if (!neigh) { 3363 pr_err("%s - failed to allocate neigh!\n", 3364 __func__); 3365 goto free_dst; 3366 } 3367 3368 if (neigh->dev->flags & IFF_LOOPBACK) { 3369 pdev = ip_dev_find(&init_net, iph->daddr); 3370 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3371 pdev, 0); 3372 pi = (struct port_info *)netdev_priv(pdev); 3373 tx_chan = cxgb4_port_chan(pdev); 3374 dev_put(pdev); 3375 } else { 3376 pdev = get_real_dev(neigh->dev); 3377 e = cxgb4_l2t_get(dev->rdev.lldi.l2t, neigh, 3378 pdev, 0); 3379 pi = (struct port_info *)netdev_priv(pdev); 3380 tx_chan = cxgb4_port_chan(pdev); 3381 } 3382 neigh_release(neigh); 3383 if (!e) { 3384 pr_err("%s - failed to allocate l2t entry!\n", 3385 __func__); 3386 goto free_dst; 3387 } 3388 3389 step = dev->rdev.lldi.nrxq / dev->rdev.lldi.nchan; 3390 rss_qid = dev->rdev.lldi.rxq_ids[pi->port_id * step]; 3391 window = (__force u16) htons((__force u16)tcph->window); 3392 3393 /* Calcuate filter portion for LE region. */ 3394 filter = (__force unsigned int) cpu_to_be32(cxgb4_select_ntuple( 3395 dev->rdev.lldi.ports[0], 3396 e)); 3397 3398 /* 3399 * Synthesize the cpl_pass_accept_req. We have everything except the 3400 * TID. Once firmware sends a reply with TID we update the TID field 3401 * in cpl and pass it through the regular cpl_pass_accept_req path. 3402 */ 3403 build_cpl_pass_accept_req(skb, stid, iph->tos); 3404 send_fw_pass_open_req(dev, skb, iph->daddr, tcph->dest, iph->saddr, 3405 tcph->source, ntohl(tcph->seq), filter, window, 3406 rss_qid, pi->port_id); 3407 cxgb4_l2t_release(e); 3408 free_dst: 3409 dst_release(dst); 3410 reject: 3411 return 0; 3412 } 3413 3414 /* 3415 * These are the real handlers that are called from a 3416 * work queue. 3417 */ 3418 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 3419 [CPL_ACT_ESTABLISH] = act_establish, 3420 [CPL_ACT_OPEN_RPL] = act_open_rpl, 3421 [CPL_RX_DATA] = rx_data, 3422 [CPL_ABORT_RPL_RSS] = abort_rpl, 3423 [CPL_ABORT_RPL] = abort_rpl, 3424 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 3425 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 3426 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 3427 [CPL_PASS_ESTABLISH] = pass_establish, 3428 [CPL_PEER_CLOSE] = peer_close, 3429 [CPL_ABORT_REQ_RSS] = peer_abort, 3430 [CPL_CLOSE_CON_RPL] = close_con_rpl, 3431 [CPL_RDMA_TERMINATE] = terminate, 3432 [CPL_FW4_ACK] = fw4_ack, 3433 [CPL_FW6_MSG] = deferred_fw6_msg, 3434 [CPL_RX_PKT] = rx_pkt 3435 }; 3436 3437 static void process_timeout(struct c4iw_ep *ep) 3438 { 3439 struct c4iw_qp_attributes attrs; 3440 int abort = 1; 3441 3442 mutex_lock(&ep->com.mutex); 3443 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 3444 ep->com.state); 3445 set_bit(TIMEDOUT, &ep->com.history); 3446 switch (ep->com.state) { 3447 case MPA_REQ_SENT: 3448 __state_set(&ep->com, ABORTING); 3449 connect_reply_upcall(ep, -ETIMEDOUT); 3450 break; 3451 case MPA_REQ_WAIT: 3452 __state_set(&ep->com, ABORTING); 3453 break; 3454 case CLOSING: 3455 case MORIBUND: 3456 if (ep->com.cm_id && ep->com.qp) { 3457 attrs.next_state = C4IW_QP_STATE_ERROR; 3458 c4iw_modify_qp(ep->com.qp->rhp, 3459 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 3460 &attrs, 1); 3461 } 3462 __state_set(&ep->com, ABORTING); 3463 close_complete_upcall(ep, -ETIMEDOUT); 3464 break; 3465 default: 3466 WARN(1, "%s unexpected state ep %p tid %u state %u\n", 3467 __func__, ep, ep->hwtid, ep->com.state); 3468 abort = 0; 3469 } 3470 mutex_unlock(&ep->com.mutex); 3471 if (abort) 3472 abort_connection(ep, NULL, GFP_KERNEL); 3473 c4iw_put_ep(&ep->com); 3474 } 3475 3476 static void process_timedout_eps(void) 3477 { 3478 struct c4iw_ep *ep; 3479 3480 spin_lock_irq(&timeout_lock); 3481 while (!list_empty(&timeout_list)) { 3482 struct list_head *tmp; 3483 3484 tmp = timeout_list.next; 3485 list_del(tmp); 3486 spin_unlock_irq(&timeout_lock); 3487 ep = list_entry(tmp, struct c4iw_ep, entry); 3488 process_timeout(ep); 3489 spin_lock_irq(&timeout_lock); 3490 } 3491 spin_unlock_irq(&timeout_lock); 3492 } 3493 3494 static void process_work(struct work_struct *work) 3495 { 3496 struct sk_buff *skb = NULL; 3497 struct c4iw_dev *dev; 3498 struct cpl_act_establish *rpl; 3499 unsigned int opcode; 3500 int ret; 3501 3502 while ((skb = skb_dequeue(&rxq))) { 3503 rpl = cplhdr(skb); 3504 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 3505 opcode = rpl->ot.opcode; 3506 3507 BUG_ON(!work_handlers[opcode]); 3508 ret = work_handlers[opcode](dev, skb); 3509 if (!ret) 3510 kfree_skb(skb); 3511 } 3512 process_timedout_eps(); 3513 } 3514 3515 static DECLARE_WORK(skb_work, process_work); 3516 3517 static void ep_timeout(unsigned long arg) 3518 { 3519 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 3520 int kickit = 0; 3521 3522 spin_lock(&timeout_lock); 3523 if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { 3524 list_add_tail(&ep->entry, &timeout_list); 3525 kickit = 1; 3526 } 3527 spin_unlock(&timeout_lock); 3528 if (kickit) 3529 queue_work(workq, &skb_work); 3530 } 3531 3532 /* 3533 * All the CM events are handled on a work queue to have a safe context. 3534 */ 3535 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 3536 { 3537 3538 /* 3539 * Save dev in the skb->cb area. 3540 */ 3541 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 3542 3543 /* 3544 * Queue the skb and schedule the worker thread. 3545 */ 3546 skb_queue_tail(&rxq, skb); 3547 queue_work(workq, &skb_work); 3548 return 0; 3549 } 3550 3551 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 3552 { 3553 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 3554 3555 if (rpl->status != CPL_ERR_NONE) { 3556 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 3557 "for tid %u\n", rpl->status, GET_TID(rpl)); 3558 } 3559 kfree_skb(skb); 3560 return 0; 3561 } 3562 3563 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 3564 { 3565 struct cpl_fw6_msg *rpl = cplhdr(skb); 3566 struct c4iw_wr_wait *wr_waitp; 3567 int ret; 3568 3569 PDBG("%s type %u\n", __func__, rpl->type); 3570 3571 switch (rpl->type) { 3572 case FW6_TYPE_WR_RPL: 3573 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 3574 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 3575 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 3576 if (wr_waitp) 3577 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 3578 kfree_skb(skb); 3579 break; 3580 case FW6_TYPE_CQE: 3581 case FW6_TYPE_OFLD_CONNECTION_WR_RPL: 3582 sched(dev, skb); 3583 break; 3584 default: 3585 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 3586 rpl->type); 3587 kfree_skb(skb); 3588 break; 3589 } 3590 return 0; 3591 } 3592 3593 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 3594 { 3595 struct cpl_abort_req_rss *req = cplhdr(skb); 3596 struct c4iw_ep *ep; 3597 struct tid_info *t = dev->rdev.lldi.tids; 3598 unsigned int tid = GET_TID(req); 3599 3600 ep = lookup_tid(t, tid); 3601 if (!ep) { 3602 printk(KERN_WARNING MOD 3603 "Abort on non-existent endpoint, tid %d\n", tid); 3604 kfree_skb(skb); 3605 return 0; 3606 } 3607 if (is_neg_adv(req->status)) { 3608 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 3609 ep->hwtid); 3610 kfree_skb(skb); 3611 return 0; 3612 } 3613 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 3614 ep->com.state); 3615 3616 /* 3617 * Wake up any threads in rdma_init() or rdma_fini(). 3618 * However, if we are on MPAv2 and want to retry with MPAv1 3619 * then, don't wake up yet. 3620 */ 3621 if (mpa_rev == 2 && !ep->tried_with_mpa_v1) { 3622 if (ep->com.state != MPA_REQ_SENT) 3623 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3624 } else 3625 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 3626 sched(dev, skb); 3627 return 0; 3628 } 3629 3630 /* 3631 * Most upcalls from the T4 Core go to sched() to 3632 * schedule the processing on a work queue. 3633 */ 3634 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 3635 [CPL_ACT_ESTABLISH] = sched, 3636 [CPL_ACT_OPEN_RPL] = sched, 3637 [CPL_RX_DATA] = sched, 3638 [CPL_ABORT_RPL_RSS] = sched, 3639 [CPL_ABORT_RPL] = sched, 3640 [CPL_PASS_OPEN_RPL] = sched, 3641 [CPL_CLOSE_LISTSRV_RPL] = sched, 3642 [CPL_PASS_ACCEPT_REQ] = sched, 3643 [CPL_PASS_ESTABLISH] = sched, 3644 [CPL_PEER_CLOSE] = sched, 3645 [CPL_CLOSE_CON_RPL] = sched, 3646 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 3647 [CPL_RDMA_TERMINATE] = sched, 3648 [CPL_FW4_ACK] = sched, 3649 [CPL_SET_TCB_RPL] = set_tcb_rpl, 3650 [CPL_FW6_MSG] = fw6_msg, 3651 [CPL_RX_PKT] = sched 3652 }; 3653 3654 int __init c4iw_cm_init(void) 3655 { 3656 spin_lock_init(&timeout_lock); 3657 skb_queue_head_init(&rxq); 3658 3659 workq = create_singlethread_workqueue("iw_cxgb4"); 3660 if (!workq) 3661 return -ENOMEM; 3662 3663 return 0; 3664 } 3665 3666 void __exit c4iw_cm_term(void) 3667 { 3668 WARN_ON(!list_empty(&timeout_list)); 3669 flush_workqueue(workq); 3670 destroy_workqueue(workq); 3671 } 3672