1 /* 2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <linux/module.h> 33 #include <linux/list.h> 34 #include <linux/workqueue.h> 35 #include <linux/skbuff.h> 36 #include <linux/timer.h> 37 #include <linux/notifier.h> 38 #include <linux/inetdevice.h> 39 #include <linux/ip.h> 40 #include <linux/tcp.h> 41 42 #include <net/neighbour.h> 43 #include <net/netevent.h> 44 #include <net/route.h> 45 46 #include "iw_cxgb4.h" 47 48 static char *states[] = { 49 "idle", 50 "listen", 51 "connecting", 52 "mpa_wait_req", 53 "mpa_req_sent", 54 "mpa_req_rcvd", 55 "mpa_rep_sent", 56 "fpdu_mode", 57 "aborting", 58 "closing", 59 "moribund", 60 "dead", 61 NULL, 62 }; 63 64 static int dack_mode = 1; 65 module_param(dack_mode, int, 0644); 66 MODULE_PARM_DESC(dack_mode, "Delayed ack mode (default=1)"); 67 68 int c4iw_max_read_depth = 8; 69 module_param(c4iw_max_read_depth, int, 0644); 70 MODULE_PARM_DESC(c4iw_max_read_depth, "Per-connection max ORD/IRD (default=8)"); 71 72 static int enable_tcp_timestamps; 73 module_param(enable_tcp_timestamps, int, 0644); 74 MODULE_PARM_DESC(enable_tcp_timestamps, "Enable tcp timestamps (default=0)"); 75 76 static int enable_tcp_sack; 77 module_param(enable_tcp_sack, int, 0644); 78 MODULE_PARM_DESC(enable_tcp_sack, "Enable tcp SACK (default=0)"); 79 80 static int enable_tcp_window_scaling = 1; 81 module_param(enable_tcp_window_scaling, int, 0644); 82 MODULE_PARM_DESC(enable_tcp_window_scaling, 83 "Enable tcp window scaling (default=1)"); 84 85 int c4iw_debug; 86 module_param(c4iw_debug, int, 0644); 87 MODULE_PARM_DESC(c4iw_debug, "Enable debug logging (default=0)"); 88 89 static int peer2peer; 90 module_param(peer2peer, int, 0644); 91 MODULE_PARM_DESC(peer2peer, "Support peer2peer ULPs (default=0)"); 92 93 static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; 94 module_param(p2p_type, int, 0644); 95 MODULE_PARM_DESC(p2p_type, "RDMAP opcode to use for the RTR message: " 96 "1=RDMA_READ 0=RDMA_WRITE (default 1)"); 97 98 static int ep_timeout_secs = 60; 99 module_param(ep_timeout_secs, int, 0644); 100 MODULE_PARM_DESC(ep_timeout_secs, "CM Endpoint operation timeout " 101 "in seconds (default=60)"); 102 103 static int mpa_rev = 1; 104 module_param(mpa_rev, int, 0644); 105 MODULE_PARM_DESC(mpa_rev, "MPA Revision, 0 supports amso1100, " 106 "1 is RFC0544 spec compliant, 2 is IETF MPA Peer Connect Draft" 107 " compliant (default=1)"); 108 109 static int markers_enabled; 110 module_param(markers_enabled, int, 0644); 111 MODULE_PARM_DESC(markers_enabled, "Enable MPA MARKERS (default(0)=disabled)"); 112 113 static int crc_enabled = 1; 114 module_param(crc_enabled, int, 0644); 115 MODULE_PARM_DESC(crc_enabled, "Enable MPA CRC (default(1)=enabled)"); 116 117 static int rcv_win = 256 * 1024; 118 module_param(rcv_win, int, 0644); 119 MODULE_PARM_DESC(rcv_win, "TCP receive window in bytes (default=256KB)"); 120 121 static int snd_win = 128 * 1024; 122 module_param(snd_win, int, 0644); 123 MODULE_PARM_DESC(snd_win, "TCP send window in bytes (default=128KB)"); 124 125 static struct workqueue_struct *workq; 126 127 static struct sk_buff_head rxq; 128 129 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp); 130 static void ep_timeout(unsigned long arg); 131 static void connect_reply_upcall(struct c4iw_ep *ep, int status); 132 133 static LIST_HEAD(timeout_list); 134 static spinlock_t timeout_lock; 135 136 static void start_ep_timer(struct c4iw_ep *ep) 137 { 138 PDBG("%s ep %p\n", __func__, ep); 139 if (timer_pending(&ep->timer)) { 140 PDBG("%s stopped / restarted timer ep %p\n", __func__, ep); 141 del_timer_sync(&ep->timer); 142 } else 143 c4iw_get_ep(&ep->com); 144 ep->timer.expires = jiffies + ep_timeout_secs * HZ; 145 ep->timer.data = (unsigned long)ep; 146 ep->timer.function = ep_timeout; 147 add_timer(&ep->timer); 148 } 149 150 static void stop_ep_timer(struct c4iw_ep *ep) 151 { 152 PDBG("%s ep %p\n", __func__, ep); 153 if (!timer_pending(&ep->timer)) { 154 printk(KERN_ERR "%s timer stopped when its not running! " 155 "ep %p state %u\n", __func__, ep, ep->com.state); 156 WARN_ON(1); 157 return; 158 } 159 del_timer_sync(&ep->timer); 160 c4iw_put_ep(&ep->com); 161 } 162 163 static int c4iw_l2t_send(struct c4iw_rdev *rdev, struct sk_buff *skb, 164 struct l2t_entry *l2e) 165 { 166 int error = 0; 167 168 if (c4iw_fatal_error(rdev)) { 169 kfree_skb(skb); 170 PDBG("%s - device in error state - dropping\n", __func__); 171 return -EIO; 172 } 173 error = cxgb4_l2t_send(rdev->lldi.ports[0], skb, l2e); 174 if (error < 0) 175 kfree_skb(skb); 176 return error < 0 ? error : 0; 177 } 178 179 int c4iw_ofld_send(struct c4iw_rdev *rdev, struct sk_buff *skb) 180 { 181 int error = 0; 182 183 if (c4iw_fatal_error(rdev)) { 184 kfree_skb(skb); 185 PDBG("%s - device in error state - dropping\n", __func__); 186 return -EIO; 187 } 188 error = cxgb4_ofld_send(rdev->lldi.ports[0], skb); 189 if (error < 0) 190 kfree_skb(skb); 191 return error < 0 ? error : 0; 192 } 193 194 static void release_tid(struct c4iw_rdev *rdev, u32 hwtid, struct sk_buff *skb) 195 { 196 struct cpl_tid_release *req; 197 198 skb = get_skb(skb, sizeof *req, GFP_KERNEL); 199 if (!skb) 200 return; 201 req = (struct cpl_tid_release *) skb_put(skb, sizeof(*req)); 202 INIT_TP_WR(req, hwtid); 203 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_TID_RELEASE, hwtid)); 204 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 205 c4iw_ofld_send(rdev, skb); 206 return; 207 } 208 209 static void set_emss(struct c4iw_ep *ep, u16 opt) 210 { 211 ep->emss = ep->com.dev->rdev.lldi.mtus[GET_TCPOPT_MSS(opt)] - 40; 212 ep->mss = ep->emss; 213 if (GET_TCPOPT_TSTAMP(opt)) 214 ep->emss -= 12; 215 if (ep->emss < 128) 216 ep->emss = 128; 217 PDBG("%s mss_idx %u mss %u emss=%u\n", __func__, GET_TCPOPT_MSS(opt), 218 ep->mss, ep->emss); 219 } 220 221 static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc) 222 { 223 enum c4iw_ep_state state; 224 225 mutex_lock(&epc->mutex); 226 state = epc->state; 227 mutex_unlock(&epc->mutex); 228 return state; 229 } 230 231 static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 232 { 233 epc->state = new; 234 } 235 236 static void state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state new) 237 { 238 mutex_lock(&epc->mutex); 239 PDBG("%s - %s -> %s\n", __func__, states[epc->state], states[new]); 240 __state_set(epc, new); 241 mutex_unlock(&epc->mutex); 242 return; 243 } 244 245 static void *alloc_ep(int size, gfp_t gfp) 246 { 247 struct c4iw_ep_common *epc; 248 249 epc = kzalloc(size, gfp); 250 if (epc) { 251 kref_init(&epc->kref); 252 mutex_init(&epc->mutex); 253 c4iw_init_wr_wait(&epc->wr_wait); 254 } 255 PDBG("%s alloc ep %p\n", __func__, epc); 256 return epc; 257 } 258 259 void _c4iw_free_ep(struct kref *kref) 260 { 261 struct c4iw_ep *ep; 262 263 ep = container_of(kref, struct c4iw_ep, com.kref); 264 PDBG("%s ep %p state %s\n", __func__, ep, states[state_read(&ep->com)]); 265 if (test_bit(RELEASE_RESOURCES, &ep->com.flags)) { 266 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 267 dst_release(ep->dst); 268 cxgb4_l2t_release(ep->l2t); 269 } 270 kfree(ep); 271 } 272 273 static void release_ep_resources(struct c4iw_ep *ep) 274 { 275 set_bit(RELEASE_RESOURCES, &ep->com.flags); 276 c4iw_put_ep(&ep->com); 277 } 278 279 static int status2errno(int status) 280 { 281 switch (status) { 282 case CPL_ERR_NONE: 283 return 0; 284 case CPL_ERR_CONN_RESET: 285 return -ECONNRESET; 286 case CPL_ERR_ARP_MISS: 287 return -EHOSTUNREACH; 288 case CPL_ERR_CONN_TIMEDOUT: 289 return -ETIMEDOUT; 290 case CPL_ERR_TCAM_FULL: 291 return -ENOMEM; 292 case CPL_ERR_CONN_EXIST: 293 return -EADDRINUSE; 294 default: 295 return -EIO; 296 } 297 } 298 299 /* 300 * Try and reuse skbs already allocated... 301 */ 302 static struct sk_buff *get_skb(struct sk_buff *skb, int len, gfp_t gfp) 303 { 304 if (skb && !skb_is_nonlinear(skb) && !skb_cloned(skb)) { 305 skb_trim(skb, 0); 306 skb_get(skb); 307 skb_reset_transport_header(skb); 308 } else { 309 skb = alloc_skb(len, gfp); 310 } 311 return skb; 312 } 313 314 static struct rtable *find_route(struct c4iw_dev *dev, __be32 local_ip, 315 __be32 peer_ip, __be16 local_port, 316 __be16 peer_port, u8 tos) 317 { 318 struct rtable *rt; 319 struct flowi4 fl4; 320 321 rt = ip_route_output_ports(&init_net, &fl4, NULL, peer_ip, local_ip, 322 peer_port, local_port, IPPROTO_TCP, 323 tos, 0); 324 if (IS_ERR(rt)) 325 return NULL; 326 return rt; 327 } 328 329 static void arp_failure_discard(void *handle, struct sk_buff *skb) 330 { 331 PDBG("%s c4iw_dev %p\n", __func__, handle); 332 kfree_skb(skb); 333 } 334 335 /* 336 * Handle an ARP failure for an active open. 337 */ 338 static void act_open_req_arp_failure(void *handle, struct sk_buff *skb) 339 { 340 printk(KERN_ERR MOD "ARP failure duing connect\n"); 341 kfree_skb(skb); 342 } 343 344 /* 345 * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 346 * and send it along. 347 */ 348 static void abort_arp_failure(void *handle, struct sk_buff *skb) 349 { 350 struct c4iw_rdev *rdev = handle; 351 struct cpl_abort_req *req = cplhdr(skb); 352 353 PDBG("%s rdev %p\n", __func__, rdev); 354 req->cmd = CPL_ABORT_NO_RST; 355 c4iw_ofld_send(rdev, skb); 356 } 357 358 static void send_flowc(struct c4iw_ep *ep, struct sk_buff *skb) 359 { 360 unsigned int flowclen = 80; 361 struct fw_flowc_wr *flowc; 362 int i; 363 364 skb = get_skb(skb, flowclen, GFP_KERNEL); 365 flowc = (struct fw_flowc_wr *)__skb_put(skb, flowclen); 366 367 flowc->op_to_nparams = cpu_to_be32(FW_WR_OP(FW_FLOWC_WR) | 368 FW_FLOWC_WR_NPARAMS(8)); 369 flowc->flowid_len16 = cpu_to_be32(FW_WR_LEN16(DIV_ROUND_UP(flowclen, 370 16)) | FW_WR_FLOWID(ep->hwtid)); 371 372 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 373 flowc->mnemval[0].val = cpu_to_be32(PCI_FUNC(ep->com.dev->rdev.lldi.pdev->devfn) << 8); 374 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 375 flowc->mnemval[1].val = cpu_to_be32(ep->tx_chan); 376 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 377 flowc->mnemval[2].val = cpu_to_be32(ep->tx_chan); 378 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 379 flowc->mnemval[3].val = cpu_to_be32(ep->rss_qid); 380 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 381 flowc->mnemval[4].val = cpu_to_be32(ep->snd_seq); 382 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 383 flowc->mnemval[5].val = cpu_to_be32(ep->rcv_seq); 384 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 385 flowc->mnemval[6].val = cpu_to_be32(snd_win); 386 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 387 flowc->mnemval[7].val = cpu_to_be32(ep->emss); 388 /* Pad WR to 16 byte boundary */ 389 flowc->mnemval[8].mnemonic = 0; 390 flowc->mnemval[8].val = 0; 391 for (i = 0; i < 9; i++) { 392 flowc->mnemval[i].r4[0] = 0; 393 flowc->mnemval[i].r4[1] = 0; 394 flowc->mnemval[i].r4[2] = 0; 395 } 396 397 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 398 c4iw_ofld_send(&ep->com.dev->rdev, skb); 399 } 400 401 static int send_halfclose(struct c4iw_ep *ep, gfp_t gfp) 402 { 403 struct cpl_close_con_req *req; 404 struct sk_buff *skb; 405 int wrlen = roundup(sizeof *req, 16); 406 407 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 408 skb = get_skb(NULL, wrlen, gfp); 409 if (!skb) { 410 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 411 return -ENOMEM; 412 } 413 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 414 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 415 req = (struct cpl_close_con_req *) skb_put(skb, wrlen); 416 memset(req, 0, wrlen); 417 INIT_TP_WR(req, ep->hwtid); 418 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, 419 ep->hwtid)); 420 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 421 } 422 423 static int send_abort(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 424 { 425 struct cpl_abort_req *req; 426 int wrlen = roundup(sizeof *req, 16); 427 428 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 429 skb = get_skb(skb, wrlen, gfp); 430 if (!skb) { 431 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 432 __func__); 433 return -ENOMEM; 434 } 435 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 436 t4_set_arp_err_handler(skb, &ep->com.dev->rdev, abort_arp_failure); 437 req = (struct cpl_abort_req *) skb_put(skb, wrlen); 438 memset(req, 0, wrlen); 439 INIT_TP_WR(req, ep->hwtid); 440 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, ep->hwtid)); 441 req->cmd = CPL_ABORT_SEND_RST; 442 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 443 } 444 445 static int send_connect(struct c4iw_ep *ep) 446 { 447 struct cpl_act_open_req *req; 448 struct sk_buff *skb; 449 u64 opt0; 450 u32 opt2; 451 unsigned int mtu_idx; 452 int wscale; 453 int wrlen = roundup(sizeof *req, 16); 454 455 PDBG("%s ep %p atid %u\n", __func__, ep, ep->atid); 456 457 skb = get_skb(NULL, wrlen, GFP_KERNEL); 458 if (!skb) { 459 printk(KERN_ERR MOD "%s - failed to alloc skb.\n", 460 __func__); 461 return -ENOMEM; 462 } 463 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 464 465 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 466 wscale = compute_wscale(rcv_win); 467 opt0 = KEEP_ALIVE(1) | 468 DELACK(1) | 469 WND_SCALE(wscale) | 470 MSS_IDX(mtu_idx) | 471 L2T_IDX(ep->l2t->idx) | 472 TX_CHAN(ep->tx_chan) | 473 SMAC_SEL(ep->smac_idx) | 474 DSCP(ep->tos) | 475 ULP_MODE(ULP_MODE_TCPDDP) | 476 RCV_BUFSIZ(rcv_win>>10); 477 opt2 = RX_CHANNEL(0) | 478 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 479 if (enable_tcp_timestamps) 480 opt2 |= TSTAMPS_EN(1); 481 if (enable_tcp_sack) 482 opt2 |= SACK_EN(1); 483 if (wscale && enable_tcp_window_scaling) 484 opt2 |= WND_SCALE_EN(1); 485 t4_set_arp_err_handler(skb, NULL, act_open_req_arp_failure); 486 487 req = (struct cpl_act_open_req *) skb_put(skb, wrlen); 488 INIT_TP_WR(req, 0); 489 OPCODE_TID(req) = cpu_to_be32( 490 MK_OPCODE_TID(CPL_ACT_OPEN_REQ, ((ep->rss_qid<<14)|ep->atid))); 491 req->local_port = ep->com.local_addr.sin_port; 492 req->peer_port = ep->com.remote_addr.sin_port; 493 req->local_ip = ep->com.local_addr.sin_addr.s_addr; 494 req->peer_ip = ep->com.remote_addr.sin_addr.s_addr; 495 req->opt0 = cpu_to_be64(opt0); 496 req->params = 0; 497 req->opt2 = cpu_to_be32(opt2); 498 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 499 } 500 501 static void send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, 502 u8 mpa_rev_to_use) 503 { 504 int mpalen, wrlen; 505 struct fw_ofld_tx_data_wr *req; 506 struct mpa_message *mpa; 507 struct mpa_v2_conn_params mpa_v2_params; 508 509 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 510 511 BUG_ON(skb_cloned(skb)); 512 513 mpalen = sizeof(*mpa) + ep->plen; 514 if (mpa_rev_to_use == 2) 515 mpalen += sizeof(struct mpa_v2_conn_params); 516 wrlen = roundup(mpalen + sizeof *req, 16); 517 skb = get_skb(skb, wrlen, GFP_KERNEL); 518 if (!skb) { 519 connect_reply_upcall(ep, -ENOMEM); 520 return; 521 } 522 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 523 524 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 525 memset(req, 0, wrlen); 526 req->op_to_immdlen = cpu_to_be32( 527 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 528 FW_WR_COMPL(1) | 529 FW_WR_IMMDLEN(mpalen)); 530 req->flowid_len16 = cpu_to_be32( 531 FW_WR_FLOWID(ep->hwtid) | 532 FW_WR_LEN16(wrlen >> 4)); 533 req->plen = cpu_to_be32(mpalen); 534 req->tunnel_to_proxy = cpu_to_be32( 535 FW_OFLD_TX_DATA_WR_FLUSH(1) | 536 FW_OFLD_TX_DATA_WR_SHOVE(1)); 537 538 mpa = (struct mpa_message *)(req + 1); 539 memcpy(mpa->key, MPA_KEY_REQ, sizeof(mpa->key)); 540 mpa->flags = (crc_enabled ? MPA_CRC : 0) | 541 (markers_enabled ? MPA_MARKERS : 0) | 542 (mpa_rev_to_use == 2 ? MPA_ENHANCED_RDMA_CONN : 0); 543 mpa->private_data_size = htons(ep->plen); 544 mpa->revision = mpa_rev_to_use; 545 if (mpa_rev_to_use == 1) { 546 ep->tried_with_mpa_v1 = 1; 547 ep->retry_with_mpa_v1 = 0; 548 } 549 550 if (mpa_rev_to_use == 2) { 551 mpa->private_data_size += 552 htons(sizeof(struct mpa_v2_conn_params)); 553 mpa_v2_params.ird = htons((u16)ep->ird); 554 mpa_v2_params.ord = htons((u16)ep->ord); 555 556 if (peer2peer) { 557 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 558 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 559 mpa_v2_params.ord |= 560 htons(MPA_V2_RDMA_WRITE_RTR); 561 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 562 mpa_v2_params.ord |= 563 htons(MPA_V2_RDMA_READ_RTR); 564 } 565 memcpy(mpa->private_data, &mpa_v2_params, 566 sizeof(struct mpa_v2_conn_params)); 567 568 if (ep->plen) 569 memcpy(mpa->private_data + 570 sizeof(struct mpa_v2_conn_params), 571 ep->mpa_pkt + sizeof(*mpa), ep->plen); 572 } else 573 if (ep->plen) 574 memcpy(mpa->private_data, 575 ep->mpa_pkt + sizeof(*mpa), ep->plen); 576 577 /* 578 * Reference the mpa skb. This ensures the data area 579 * will remain in memory until the hw acks the tx. 580 * Function fw4_ack() will deref it. 581 */ 582 skb_get(skb); 583 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 584 BUG_ON(ep->mpa_skb); 585 ep->mpa_skb = skb; 586 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 587 start_ep_timer(ep); 588 state_set(&ep->com, MPA_REQ_SENT); 589 ep->mpa_attr.initiator = 1; 590 return; 591 } 592 593 static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) 594 { 595 int mpalen, wrlen; 596 struct fw_ofld_tx_data_wr *req; 597 struct mpa_message *mpa; 598 struct sk_buff *skb; 599 struct mpa_v2_conn_params mpa_v2_params; 600 601 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 602 603 mpalen = sizeof(*mpa) + plen; 604 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 605 mpalen += sizeof(struct mpa_v2_conn_params); 606 wrlen = roundup(mpalen + sizeof *req, 16); 607 608 skb = get_skb(NULL, wrlen, GFP_KERNEL); 609 if (!skb) { 610 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 611 return -ENOMEM; 612 } 613 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 614 615 req = (struct fw_ofld_tx_data_wr *)skb_put(skb, wrlen); 616 memset(req, 0, wrlen); 617 req->op_to_immdlen = cpu_to_be32( 618 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 619 FW_WR_COMPL(1) | 620 FW_WR_IMMDLEN(mpalen)); 621 req->flowid_len16 = cpu_to_be32( 622 FW_WR_FLOWID(ep->hwtid) | 623 FW_WR_LEN16(wrlen >> 4)); 624 req->plen = cpu_to_be32(mpalen); 625 req->tunnel_to_proxy = cpu_to_be32( 626 FW_OFLD_TX_DATA_WR_FLUSH(1) | 627 FW_OFLD_TX_DATA_WR_SHOVE(1)); 628 629 mpa = (struct mpa_message *)(req + 1); 630 memset(mpa, 0, sizeof(*mpa)); 631 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 632 mpa->flags = MPA_REJECT; 633 mpa->revision = mpa_rev; 634 mpa->private_data_size = htons(plen); 635 636 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 637 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 638 mpa->private_data_size += 639 htons(sizeof(struct mpa_v2_conn_params)); 640 mpa_v2_params.ird = htons(((u16)ep->ird) | 641 (peer2peer ? MPA_V2_PEER2PEER_MODEL : 642 0)); 643 mpa_v2_params.ord = htons(((u16)ep->ord) | (peer2peer ? 644 (p2p_type == 645 FW_RI_INIT_P2PTYPE_RDMA_WRITE ? 646 MPA_V2_RDMA_WRITE_RTR : p2p_type == 647 FW_RI_INIT_P2PTYPE_READ_REQ ? 648 MPA_V2_RDMA_READ_RTR : 0) : 0)); 649 memcpy(mpa->private_data, &mpa_v2_params, 650 sizeof(struct mpa_v2_conn_params)); 651 652 if (ep->plen) 653 memcpy(mpa->private_data + 654 sizeof(struct mpa_v2_conn_params), pdata, plen); 655 } else 656 if (plen) 657 memcpy(mpa->private_data, pdata, plen); 658 659 /* 660 * Reference the mpa skb again. This ensures the data area 661 * will remain in memory until the hw acks the tx. 662 * Function fw4_ack() will deref it. 663 */ 664 skb_get(skb); 665 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 666 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 667 BUG_ON(ep->mpa_skb); 668 ep->mpa_skb = skb; 669 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 670 } 671 672 static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) 673 { 674 int mpalen, wrlen; 675 struct fw_ofld_tx_data_wr *req; 676 struct mpa_message *mpa; 677 struct sk_buff *skb; 678 struct mpa_v2_conn_params mpa_v2_params; 679 680 PDBG("%s ep %p tid %u pd_len %d\n", __func__, ep, ep->hwtid, ep->plen); 681 682 mpalen = sizeof(*mpa) + plen; 683 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) 684 mpalen += sizeof(struct mpa_v2_conn_params); 685 wrlen = roundup(mpalen + sizeof *req, 16); 686 687 skb = get_skb(NULL, wrlen, GFP_KERNEL); 688 if (!skb) { 689 printk(KERN_ERR MOD "%s - cannot alloc skb!\n", __func__); 690 return -ENOMEM; 691 } 692 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 693 694 req = (struct fw_ofld_tx_data_wr *) skb_put(skb, wrlen); 695 memset(req, 0, wrlen); 696 req->op_to_immdlen = cpu_to_be32( 697 FW_WR_OP(FW_OFLD_TX_DATA_WR) | 698 FW_WR_COMPL(1) | 699 FW_WR_IMMDLEN(mpalen)); 700 req->flowid_len16 = cpu_to_be32( 701 FW_WR_FLOWID(ep->hwtid) | 702 FW_WR_LEN16(wrlen >> 4)); 703 req->plen = cpu_to_be32(mpalen); 704 req->tunnel_to_proxy = cpu_to_be32( 705 FW_OFLD_TX_DATA_WR_FLUSH(1) | 706 FW_OFLD_TX_DATA_WR_SHOVE(1)); 707 708 mpa = (struct mpa_message *)(req + 1); 709 memset(mpa, 0, sizeof(*mpa)); 710 memcpy(mpa->key, MPA_KEY_REP, sizeof(mpa->key)); 711 mpa->flags = (ep->mpa_attr.crc_enabled ? MPA_CRC : 0) | 712 (markers_enabled ? MPA_MARKERS : 0); 713 mpa->revision = ep->mpa_attr.version; 714 mpa->private_data_size = htons(plen); 715 716 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 717 mpa->flags |= MPA_ENHANCED_RDMA_CONN; 718 mpa->private_data_size += 719 htons(sizeof(struct mpa_v2_conn_params)); 720 mpa_v2_params.ird = htons((u16)ep->ird); 721 mpa_v2_params.ord = htons((u16)ep->ord); 722 if (peer2peer && (ep->mpa_attr.p2p_type != 723 FW_RI_INIT_P2PTYPE_DISABLED)) { 724 mpa_v2_params.ird |= htons(MPA_V2_PEER2PEER_MODEL); 725 726 if (p2p_type == FW_RI_INIT_P2PTYPE_RDMA_WRITE) 727 mpa_v2_params.ord |= 728 htons(MPA_V2_RDMA_WRITE_RTR); 729 else if (p2p_type == FW_RI_INIT_P2PTYPE_READ_REQ) 730 mpa_v2_params.ord |= 731 htons(MPA_V2_RDMA_READ_RTR); 732 } 733 734 memcpy(mpa->private_data, &mpa_v2_params, 735 sizeof(struct mpa_v2_conn_params)); 736 737 if (ep->plen) 738 memcpy(mpa->private_data + 739 sizeof(struct mpa_v2_conn_params), pdata, plen); 740 } else 741 if (plen) 742 memcpy(mpa->private_data, pdata, plen); 743 744 /* 745 * Reference the mpa skb. This ensures the data area 746 * will remain in memory until the hw acks the tx. 747 * Function fw4_ack() will deref it. 748 */ 749 skb_get(skb); 750 t4_set_arp_err_handler(skb, NULL, arp_failure_discard); 751 ep->mpa_skb = skb; 752 state_set(&ep->com, MPA_REP_SENT); 753 return c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 754 } 755 756 static int act_establish(struct c4iw_dev *dev, struct sk_buff *skb) 757 { 758 struct c4iw_ep *ep; 759 struct cpl_act_establish *req = cplhdr(skb); 760 unsigned int tid = GET_TID(req); 761 unsigned int atid = GET_TID_TID(ntohl(req->tos_atid)); 762 struct tid_info *t = dev->rdev.lldi.tids; 763 764 ep = lookup_atid(t, atid); 765 766 PDBG("%s ep %p tid %u snd_isn %u rcv_isn %u\n", __func__, ep, tid, 767 be32_to_cpu(req->snd_isn), be32_to_cpu(req->rcv_isn)); 768 769 dst_confirm(ep->dst); 770 771 /* setup the hwtid for this connection */ 772 ep->hwtid = tid; 773 cxgb4_insert_tid(t, ep, tid); 774 775 ep->snd_seq = be32_to_cpu(req->snd_isn); 776 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 777 778 set_emss(ep, ntohs(req->tcp_opt)); 779 780 /* dealloc the atid */ 781 cxgb4_free_atid(t, atid); 782 783 /* start MPA negotiation */ 784 send_flowc(ep, NULL); 785 if (ep->retry_with_mpa_v1) 786 send_mpa_req(ep, skb, 1); 787 else 788 send_mpa_req(ep, skb, mpa_rev); 789 790 return 0; 791 } 792 793 static void close_complete_upcall(struct c4iw_ep *ep) 794 { 795 struct iw_cm_event event; 796 797 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 798 memset(&event, 0, sizeof(event)); 799 event.event = IW_CM_EVENT_CLOSE; 800 if (ep->com.cm_id) { 801 PDBG("close complete delivered ep %p cm_id %p tid %u\n", 802 ep, ep->com.cm_id, ep->hwtid); 803 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 804 ep->com.cm_id->rem_ref(ep->com.cm_id); 805 ep->com.cm_id = NULL; 806 ep->com.qp = NULL; 807 } 808 } 809 810 static int abort_connection(struct c4iw_ep *ep, struct sk_buff *skb, gfp_t gfp) 811 { 812 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 813 close_complete_upcall(ep); 814 state_set(&ep->com, ABORTING); 815 return send_abort(ep, skb, gfp); 816 } 817 818 static void peer_close_upcall(struct c4iw_ep *ep) 819 { 820 struct iw_cm_event event; 821 822 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 823 memset(&event, 0, sizeof(event)); 824 event.event = IW_CM_EVENT_DISCONNECT; 825 if (ep->com.cm_id) { 826 PDBG("peer close delivered ep %p cm_id %p tid %u\n", 827 ep, ep->com.cm_id, ep->hwtid); 828 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 829 } 830 } 831 832 static void peer_abort_upcall(struct c4iw_ep *ep) 833 { 834 struct iw_cm_event event; 835 836 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 837 memset(&event, 0, sizeof(event)); 838 event.event = IW_CM_EVENT_CLOSE; 839 event.status = -ECONNRESET; 840 if (ep->com.cm_id) { 841 PDBG("abort delivered ep %p cm_id %p tid %u\n", ep, 842 ep->com.cm_id, ep->hwtid); 843 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 844 ep->com.cm_id->rem_ref(ep->com.cm_id); 845 ep->com.cm_id = NULL; 846 ep->com.qp = NULL; 847 } 848 } 849 850 static void connect_reply_upcall(struct c4iw_ep *ep, int status) 851 { 852 struct iw_cm_event event; 853 854 PDBG("%s ep %p tid %u status %d\n", __func__, ep, ep->hwtid, status); 855 memset(&event, 0, sizeof(event)); 856 event.event = IW_CM_EVENT_CONNECT_REPLY; 857 event.status = status; 858 event.local_addr = ep->com.local_addr; 859 event.remote_addr = ep->com.remote_addr; 860 861 if ((status == 0) || (status == -ECONNREFUSED)) { 862 if (!ep->tried_with_mpa_v1) { 863 /* this means MPA_v2 is used */ 864 event.private_data_len = ep->plen - 865 sizeof(struct mpa_v2_conn_params); 866 event.private_data = ep->mpa_pkt + 867 sizeof(struct mpa_message) + 868 sizeof(struct mpa_v2_conn_params); 869 } else { 870 /* this means MPA_v1 is used */ 871 event.private_data_len = ep->plen; 872 event.private_data = ep->mpa_pkt + 873 sizeof(struct mpa_message); 874 } 875 } 876 877 PDBG("%s ep %p tid %u status %d\n", __func__, ep, 878 ep->hwtid, status); 879 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 880 881 if (status < 0) { 882 ep->com.cm_id->rem_ref(ep->com.cm_id); 883 ep->com.cm_id = NULL; 884 ep->com.qp = NULL; 885 } 886 } 887 888 static void connect_request_upcall(struct c4iw_ep *ep) 889 { 890 struct iw_cm_event event; 891 892 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 893 memset(&event, 0, sizeof(event)); 894 event.event = IW_CM_EVENT_CONNECT_REQUEST; 895 event.local_addr = ep->com.local_addr; 896 event.remote_addr = ep->com.remote_addr; 897 event.provider_data = ep; 898 if (!ep->tried_with_mpa_v1) { 899 /* this means MPA_v2 is used */ 900 event.ord = ep->ord; 901 event.ird = ep->ird; 902 event.private_data_len = ep->plen - 903 sizeof(struct mpa_v2_conn_params); 904 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message) + 905 sizeof(struct mpa_v2_conn_params); 906 } else { 907 /* this means MPA_v1 is used. Send max supported */ 908 event.ord = c4iw_max_read_depth; 909 event.ird = c4iw_max_read_depth; 910 event.private_data_len = ep->plen; 911 event.private_data = ep->mpa_pkt + sizeof(struct mpa_message); 912 } 913 if (state_read(&ep->parent_ep->com) != DEAD) { 914 c4iw_get_ep(&ep->com); 915 ep->parent_ep->com.cm_id->event_handler( 916 ep->parent_ep->com.cm_id, 917 &event); 918 } 919 c4iw_put_ep(&ep->parent_ep->com); 920 ep->parent_ep = NULL; 921 } 922 923 static void established_upcall(struct c4iw_ep *ep) 924 { 925 struct iw_cm_event event; 926 927 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 928 memset(&event, 0, sizeof(event)); 929 event.event = IW_CM_EVENT_ESTABLISHED; 930 event.ird = ep->ird; 931 event.ord = ep->ord; 932 if (ep->com.cm_id) { 933 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 934 ep->com.cm_id->event_handler(ep->com.cm_id, &event); 935 } 936 } 937 938 static int update_rx_credits(struct c4iw_ep *ep, u32 credits) 939 { 940 struct cpl_rx_data_ack *req; 941 struct sk_buff *skb; 942 int wrlen = roundup(sizeof *req, 16); 943 944 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 945 skb = get_skb(NULL, wrlen, GFP_KERNEL); 946 if (!skb) { 947 printk(KERN_ERR MOD "update_rx_credits - cannot alloc skb!\n"); 948 return 0; 949 } 950 951 req = (struct cpl_rx_data_ack *) skb_put(skb, wrlen); 952 memset(req, 0, wrlen); 953 INIT_TP_WR(req, ep->hwtid); 954 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 955 ep->hwtid)); 956 req->credit_dack = cpu_to_be32(credits | RX_FORCE_ACK(1) | 957 F_RX_DACK_CHANGE | 958 V_RX_DACK_MODE(dack_mode)); 959 set_wr_txq(skb, CPL_PRIORITY_ACK, ep->ctrlq_idx); 960 c4iw_ofld_send(&ep->com.dev->rdev, skb); 961 return credits; 962 } 963 964 static void process_mpa_reply(struct c4iw_ep *ep, struct sk_buff *skb) 965 { 966 struct mpa_message *mpa; 967 struct mpa_v2_conn_params *mpa_v2_params; 968 u16 plen; 969 u16 resp_ird, resp_ord; 970 u8 rtr_mismatch = 0, insuff_ird = 0; 971 struct c4iw_qp_attributes attrs; 972 enum c4iw_qp_attr_mask mask; 973 int err; 974 975 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 976 977 /* 978 * Stop mpa timer. If it expired, then the state has 979 * changed and we bail since ep_timeout already aborted 980 * the connection. 981 */ 982 stop_ep_timer(ep); 983 if (state_read(&ep->com) != MPA_REQ_SENT) 984 return; 985 986 /* 987 * If we get more than the supported amount of private data 988 * then we must fail this connection. 989 */ 990 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 991 err = -EINVAL; 992 goto err; 993 } 994 995 /* 996 * copy the new data into our accumulation buffer. 997 */ 998 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 999 skb->len); 1000 ep->mpa_pkt_len += skb->len; 1001 1002 /* 1003 * if we don't even have the mpa message, then bail. 1004 */ 1005 if (ep->mpa_pkt_len < sizeof(*mpa)) 1006 return; 1007 mpa = (struct mpa_message *) ep->mpa_pkt; 1008 1009 /* Validate MPA header. */ 1010 if (mpa->revision > mpa_rev) { 1011 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1012 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1013 err = -EPROTO; 1014 goto err; 1015 } 1016 if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { 1017 err = -EPROTO; 1018 goto err; 1019 } 1020 1021 plen = ntohs(mpa->private_data_size); 1022 1023 /* 1024 * Fail if there's too much private data. 1025 */ 1026 if (plen > MPA_MAX_PRIVATE_DATA) { 1027 err = -EPROTO; 1028 goto err; 1029 } 1030 1031 /* 1032 * If plen does not account for pkt size 1033 */ 1034 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1035 err = -EPROTO; 1036 goto err; 1037 } 1038 1039 ep->plen = (u8) plen; 1040 1041 /* 1042 * If we don't have all the pdata yet, then bail. 1043 * We'll continue process when more data arrives. 1044 */ 1045 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1046 return; 1047 1048 if (mpa->flags & MPA_REJECT) { 1049 err = -ECONNREFUSED; 1050 goto err; 1051 } 1052 1053 /* 1054 * If we get here we have accumulated the entire mpa 1055 * start reply message including private data. And 1056 * the MPA header is valid. 1057 */ 1058 state_set(&ep->com, FPDU_MODE); 1059 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1060 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1061 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1062 ep->mpa_attr.version = mpa->revision; 1063 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1064 1065 if (mpa->revision == 2) { 1066 ep->mpa_attr.enhanced_rdma_conn = 1067 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1068 if (ep->mpa_attr.enhanced_rdma_conn) { 1069 mpa_v2_params = (struct mpa_v2_conn_params *) 1070 (ep->mpa_pkt + sizeof(*mpa)); 1071 resp_ird = ntohs(mpa_v2_params->ird) & 1072 MPA_V2_IRD_ORD_MASK; 1073 resp_ord = ntohs(mpa_v2_params->ord) & 1074 MPA_V2_IRD_ORD_MASK; 1075 1076 /* 1077 * This is a double-check. Ideally, below checks are 1078 * not required since ird/ord stuff has been taken 1079 * care of in c4iw_accept_cr 1080 */ 1081 if ((ep->ird < resp_ord) || (ep->ord > resp_ird)) { 1082 err = -ENOMEM; 1083 ep->ird = resp_ord; 1084 ep->ord = resp_ird; 1085 insuff_ird = 1; 1086 } 1087 1088 if (ntohs(mpa_v2_params->ird) & 1089 MPA_V2_PEER2PEER_MODEL) { 1090 if (ntohs(mpa_v2_params->ord) & 1091 MPA_V2_RDMA_WRITE_RTR) 1092 ep->mpa_attr.p2p_type = 1093 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1094 else if (ntohs(mpa_v2_params->ord) & 1095 MPA_V2_RDMA_READ_RTR) 1096 ep->mpa_attr.p2p_type = 1097 FW_RI_INIT_P2PTYPE_READ_REQ; 1098 } 1099 } 1100 } else if (mpa->revision == 1) 1101 if (peer2peer) 1102 ep->mpa_attr.p2p_type = p2p_type; 1103 1104 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1105 "xmit_marker_enabled=%d, version=%d p2p_type=%d local-p2p_type = " 1106 "%d\n", __func__, ep->mpa_attr.crc_enabled, 1107 ep->mpa_attr.recv_marker_enabled, 1108 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1109 ep->mpa_attr.p2p_type, p2p_type); 1110 1111 /* 1112 * If responder's RTR does not match with that of initiator, assign 1113 * FW_RI_INIT_P2PTYPE_DISABLED in mpa attributes so that RTR is not 1114 * generated when moving QP to RTS state. 1115 * A TERM message will be sent after QP has moved to RTS state 1116 */ 1117 if ((ep->mpa_attr.version == 2) && peer2peer && 1118 (ep->mpa_attr.p2p_type != p2p_type)) { 1119 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1120 rtr_mismatch = 1; 1121 } 1122 1123 attrs.mpa_attr = ep->mpa_attr; 1124 attrs.max_ird = ep->ird; 1125 attrs.max_ord = ep->ord; 1126 attrs.llp_stream_handle = ep; 1127 attrs.next_state = C4IW_QP_STATE_RTS; 1128 1129 mask = C4IW_QP_ATTR_NEXT_STATE | 1130 C4IW_QP_ATTR_LLP_STREAM_HANDLE | C4IW_QP_ATTR_MPA_ATTR | 1131 C4IW_QP_ATTR_MAX_IRD | C4IW_QP_ATTR_MAX_ORD; 1132 1133 /* bind QP and TID with INIT_WR */ 1134 err = c4iw_modify_qp(ep->com.qp->rhp, 1135 ep->com.qp, mask, &attrs, 1); 1136 if (err) 1137 goto err; 1138 1139 /* 1140 * If responder's RTR requirement did not match with what initiator 1141 * supports, generate TERM message 1142 */ 1143 if (rtr_mismatch) { 1144 printk(KERN_ERR "%s: RTR mismatch, sending TERM\n", __func__); 1145 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1146 attrs.ecode = MPA_NOMATCH_RTR; 1147 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1148 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1149 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1150 err = -ENOMEM; 1151 goto out; 1152 } 1153 1154 /* 1155 * Generate TERM if initiator IRD is not sufficient for responder 1156 * provided ORD. Currently, we do the same behaviour even when 1157 * responder provided IRD is also not sufficient as regards to 1158 * initiator ORD. 1159 */ 1160 if (insuff_ird) { 1161 printk(KERN_ERR "%s: Insufficient IRD, sending TERM\n", 1162 __func__); 1163 attrs.layer_etype = LAYER_MPA | DDP_LLP; 1164 attrs.ecode = MPA_INSUFF_IRD; 1165 attrs.next_state = C4IW_QP_STATE_TERMINATE; 1166 err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1167 C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); 1168 err = -ENOMEM; 1169 goto out; 1170 } 1171 goto out; 1172 err: 1173 state_set(&ep->com, ABORTING); 1174 send_abort(ep, skb, GFP_KERNEL); 1175 out: 1176 connect_reply_upcall(ep, err); 1177 return; 1178 } 1179 1180 static void process_mpa_request(struct c4iw_ep *ep, struct sk_buff *skb) 1181 { 1182 struct mpa_message *mpa; 1183 struct mpa_v2_conn_params *mpa_v2_params; 1184 u16 plen; 1185 1186 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1187 1188 if (state_read(&ep->com) != MPA_REQ_WAIT) 1189 return; 1190 1191 /* 1192 * If we get more than the supported amount of private data 1193 * then we must fail this connection. 1194 */ 1195 if (ep->mpa_pkt_len + skb->len > sizeof(ep->mpa_pkt)) { 1196 stop_ep_timer(ep); 1197 abort_connection(ep, skb, GFP_KERNEL); 1198 return; 1199 } 1200 1201 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1202 1203 /* 1204 * Copy the new data into our accumulation buffer. 1205 */ 1206 skb_copy_from_linear_data(skb, &(ep->mpa_pkt[ep->mpa_pkt_len]), 1207 skb->len); 1208 ep->mpa_pkt_len += skb->len; 1209 1210 /* 1211 * If we don't even have the mpa message, then bail. 1212 * We'll continue process when more data arrives. 1213 */ 1214 if (ep->mpa_pkt_len < sizeof(*mpa)) 1215 return; 1216 1217 PDBG("%s enter (%s line %u)\n", __func__, __FILE__, __LINE__); 1218 stop_ep_timer(ep); 1219 mpa = (struct mpa_message *) ep->mpa_pkt; 1220 1221 /* 1222 * Validate MPA Header. 1223 */ 1224 if (mpa->revision > mpa_rev) { 1225 printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d," 1226 " Received = %d\n", __func__, mpa_rev, mpa->revision); 1227 abort_connection(ep, skb, GFP_KERNEL); 1228 return; 1229 } 1230 1231 if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) { 1232 abort_connection(ep, skb, GFP_KERNEL); 1233 return; 1234 } 1235 1236 plen = ntohs(mpa->private_data_size); 1237 1238 /* 1239 * Fail if there's too much private data. 1240 */ 1241 if (plen > MPA_MAX_PRIVATE_DATA) { 1242 abort_connection(ep, skb, GFP_KERNEL); 1243 return; 1244 } 1245 1246 /* 1247 * If plen does not account for pkt size 1248 */ 1249 if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { 1250 abort_connection(ep, skb, GFP_KERNEL); 1251 return; 1252 } 1253 ep->plen = (u8) plen; 1254 1255 /* 1256 * If we don't have all the pdata yet, then bail. 1257 */ 1258 if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) 1259 return; 1260 1261 /* 1262 * If we get here we have accumulated the entire mpa 1263 * start reply message including private data. 1264 */ 1265 ep->mpa_attr.initiator = 0; 1266 ep->mpa_attr.crc_enabled = (mpa->flags & MPA_CRC) | crc_enabled ? 1 : 0; 1267 ep->mpa_attr.recv_marker_enabled = markers_enabled; 1268 ep->mpa_attr.xmit_marker_enabled = mpa->flags & MPA_MARKERS ? 1 : 0; 1269 ep->mpa_attr.version = mpa->revision; 1270 if (mpa->revision == 1) 1271 ep->tried_with_mpa_v1 = 1; 1272 ep->mpa_attr.p2p_type = FW_RI_INIT_P2PTYPE_DISABLED; 1273 1274 if (mpa->revision == 2) { 1275 ep->mpa_attr.enhanced_rdma_conn = 1276 mpa->flags & MPA_ENHANCED_RDMA_CONN ? 1 : 0; 1277 if (ep->mpa_attr.enhanced_rdma_conn) { 1278 mpa_v2_params = (struct mpa_v2_conn_params *) 1279 (ep->mpa_pkt + sizeof(*mpa)); 1280 ep->ird = ntohs(mpa_v2_params->ird) & 1281 MPA_V2_IRD_ORD_MASK; 1282 ep->ord = ntohs(mpa_v2_params->ord) & 1283 MPA_V2_IRD_ORD_MASK; 1284 if (ntohs(mpa_v2_params->ird) & MPA_V2_PEER2PEER_MODEL) 1285 if (peer2peer) { 1286 if (ntohs(mpa_v2_params->ord) & 1287 MPA_V2_RDMA_WRITE_RTR) 1288 ep->mpa_attr.p2p_type = 1289 FW_RI_INIT_P2PTYPE_RDMA_WRITE; 1290 else if (ntohs(mpa_v2_params->ord) & 1291 MPA_V2_RDMA_READ_RTR) 1292 ep->mpa_attr.p2p_type = 1293 FW_RI_INIT_P2PTYPE_READ_REQ; 1294 } 1295 } 1296 } else if (mpa->revision == 1) 1297 if (peer2peer) 1298 ep->mpa_attr.p2p_type = p2p_type; 1299 1300 PDBG("%s - crc_enabled=%d, recv_marker_enabled=%d, " 1301 "xmit_marker_enabled=%d, version=%d p2p_type=%d\n", __func__, 1302 ep->mpa_attr.crc_enabled, ep->mpa_attr.recv_marker_enabled, 1303 ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version, 1304 ep->mpa_attr.p2p_type); 1305 1306 state_set(&ep->com, MPA_REQ_RCVD); 1307 1308 /* drive upcall */ 1309 connect_request_upcall(ep); 1310 return; 1311 } 1312 1313 static int rx_data(struct c4iw_dev *dev, struct sk_buff *skb) 1314 { 1315 struct c4iw_ep *ep; 1316 struct cpl_rx_data *hdr = cplhdr(skb); 1317 unsigned int dlen = ntohs(hdr->len); 1318 unsigned int tid = GET_TID(hdr); 1319 struct tid_info *t = dev->rdev.lldi.tids; 1320 1321 ep = lookup_tid(t, tid); 1322 PDBG("%s ep %p tid %u dlen %u\n", __func__, ep, ep->hwtid, dlen); 1323 skb_pull(skb, sizeof(*hdr)); 1324 skb_trim(skb, dlen); 1325 1326 ep->rcv_seq += dlen; 1327 BUG_ON(ep->rcv_seq != (ntohl(hdr->seq) + dlen)); 1328 1329 /* update RX credits */ 1330 update_rx_credits(ep, dlen); 1331 1332 switch (state_read(&ep->com)) { 1333 case MPA_REQ_SENT: 1334 process_mpa_reply(ep, skb); 1335 break; 1336 case MPA_REQ_WAIT: 1337 process_mpa_request(ep, skb); 1338 break; 1339 case MPA_REP_SENT: 1340 break; 1341 default: 1342 printk(KERN_ERR MOD "%s Unexpected streaming data." 1343 " ep %p state %d tid %u\n", 1344 __func__, ep, state_read(&ep->com), ep->hwtid); 1345 1346 /* 1347 * The ep will timeout and inform the ULP of the failure. 1348 * See ep_timeout(). 1349 */ 1350 break; 1351 } 1352 return 0; 1353 } 1354 1355 static int abort_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1356 { 1357 struct c4iw_ep *ep; 1358 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 1359 int release = 0; 1360 unsigned int tid = GET_TID(rpl); 1361 struct tid_info *t = dev->rdev.lldi.tids; 1362 1363 ep = lookup_tid(t, tid); 1364 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1365 BUG_ON(!ep); 1366 mutex_lock(&ep->com.mutex); 1367 switch (ep->com.state) { 1368 case ABORTING: 1369 __state_set(&ep->com, DEAD); 1370 release = 1; 1371 break; 1372 default: 1373 printk(KERN_ERR "%s ep %p state %d\n", 1374 __func__, ep, ep->com.state); 1375 break; 1376 } 1377 mutex_unlock(&ep->com.mutex); 1378 1379 if (release) 1380 release_ep_resources(ep); 1381 return 0; 1382 } 1383 1384 /* 1385 * Return whether a failed active open has allocated a TID 1386 */ 1387 static inline int act_open_has_tid(int status) 1388 { 1389 return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1390 status != CPL_ERR_ARP_MISS; 1391 } 1392 1393 static int act_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1394 { 1395 struct c4iw_ep *ep; 1396 struct cpl_act_open_rpl *rpl = cplhdr(skb); 1397 unsigned int atid = GET_TID_TID(GET_AOPEN_ATID( 1398 ntohl(rpl->atid_status))); 1399 struct tid_info *t = dev->rdev.lldi.tids; 1400 int status = GET_AOPEN_STATUS(ntohl(rpl->atid_status)); 1401 1402 ep = lookup_atid(t, atid); 1403 1404 PDBG("%s ep %p atid %u status %u errno %d\n", __func__, ep, atid, 1405 status, status2errno(status)); 1406 1407 if (status == CPL_ERR_RTX_NEG_ADVICE) { 1408 printk(KERN_WARNING MOD "Connection problems for atid %u\n", 1409 atid); 1410 return 0; 1411 } 1412 1413 connect_reply_upcall(ep, status2errno(status)); 1414 state_set(&ep->com, DEAD); 1415 1416 if (status && act_open_has_tid(status)) 1417 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, GET_TID(rpl)); 1418 1419 cxgb4_free_atid(t, atid); 1420 dst_release(ep->dst); 1421 cxgb4_l2t_release(ep->l2t); 1422 c4iw_put_ep(&ep->com); 1423 1424 return 0; 1425 } 1426 1427 static int pass_open_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1428 { 1429 struct cpl_pass_open_rpl *rpl = cplhdr(skb); 1430 struct tid_info *t = dev->rdev.lldi.tids; 1431 unsigned int stid = GET_TID(rpl); 1432 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1433 1434 if (!ep) { 1435 printk(KERN_ERR MOD "stid %d lookup failure!\n", stid); 1436 return 0; 1437 } 1438 PDBG("%s ep %p status %d error %d\n", __func__, ep, 1439 rpl->status, status2errno(rpl->status)); 1440 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1441 1442 return 0; 1443 } 1444 1445 static int listen_stop(struct c4iw_listen_ep *ep) 1446 { 1447 struct sk_buff *skb; 1448 struct cpl_close_listsvr_req *req; 1449 1450 PDBG("%s ep %p\n", __func__, ep); 1451 skb = get_skb(NULL, sizeof(*req), GFP_KERNEL); 1452 if (!skb) { 1453 printk(KERN_ERR MOD "%s - failed to alloc skb\n", __func__); 1454 return -ENOMEM; 1455 } 1456 req = (struct cpl_close_listsvr_req *) skb_put(skb, sizeof(*req)); 1457 INIT_TP_WR(req, 0); 1458 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, 1459 ep->stid)); 1460 req->reply_ctrl = cpu_to_be16( 1461 QUEUENO(ep->com.dev->rdev.lldi.rxq_ids[0])); 1462 set_wr_txq(skb, CPL_PRIORITY_SETUP, 0); 1463 return c4iw_ofld_send(&ep->com.dev->rdev, skb); 1464 } 1465 1466 static int close_listsrv_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 1467 { 1468 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb); 1469 struct tid_info *t = dev->rdev.lldi.tids; 1470 unsigned int stid = GET_TID(rpl); 1471 struct c4iw_listen_ep *ep = lookup_stid(t, stid); 1472 1473 PDBG("%s ep %p\n", __func__, ep); 1474 c4iw_wake_up(&ep->com.wr_wait, status2errno(rpl->status)); 1475 return 0; 1476 } 1477 1478 static void accept_cr(struct c4iw_ep *ep, __be32 peer_ip, struct sk_buff *skb, 1479 struct cpl_pass_accept_req *req) 1480 { 1481 struct cpl_pass_accept_rpl *rpl; 1482 unsigned int mtu_idx; 1483 u64 opt0; 1484 u32 opt2; 1485 int wscale; 1486 1487 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1488 BUG_ON(skb_cloned(skb)); 1489 skb_trim(skb, sizeof(*rpl)); 1490 skb_get(skb); 1491 cxgb4_best_mtu(ep->com.dev->rdev.lldi.mtus, ep->mtu, &mtu_idx); 1492 wscale = compute_wscale(rcv_win); 1493 opt0 = KEEP_ALIVE(1) | 1494 DELACK(1) | 1495 WND_SCALE(wscale) | 1496 MSS_IDX(mtu_idx) | 1497 L2T_IDX(ep->l2t->idx) | 1498 TX_CHAN(ep->tx_chan) | 1499 SMAC_SEL(ep->smac_idx) | 1500 DSCP(ep->tos) | 1501 ULP_MODE(ULP_MODE_TCPDDP) | 1502 RCV_BUFSIZ(rcv_win>>10); 1503 opt2 = RX_CHANNEL(0) | 1504 RSS_QUEUE_VALID | RSS_QUEUE(ep->rss_qid); 1505 1506 if (enable_tcp_timestamps && req->tcpopt.tstamp) 1507 opt2 |= TSTAMPS_EN(1); 1508 if (enable_tcp_sack && req->tcpopt.sack) 1509 opt2 |= SACK_EN(1); 1510 if (wscale && enable_tcp_window_scaling) 1511 opt2 |= WND_SCALE_EN(1); 1512 1513 rpl = cplhdr(skb); 1514 INIT_TP_WR(rpl, ep->hwtid); 1515 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, 1516 ep->hwtid)); 1517 rpl->opt0 = cpu_to_be64(opt0); 1518 rpl->opt2 = cpu_to_be32(opt2); 1519 set_wr_txq(skb, CPL_PRIORITY_SETUP, ep->ctrlq_idx); 1520 c4iw_l2t_send(&ep->com.dev->rdev, skb, ep->l2t); 1521 1522 return; 1523 } 1524 1525 static void reject_cr(struct c4iw_dev *dev, u32 hwtid, __be32 peer_ip, 1526 struct sk_buff *skb) 1527 { 1528 PDBG("%s c4iw_dev %p tid %u peer_ip %x\n", __func__, dev, hwtid, 1529 peer_ip); 1530 BUG_ON(skb_cloned(skb)); 1531 skb_trim(skb, sizeof(struct cpl_tid_release)); 1532 skb_get(skb); 1533 release_tid(&dev->rdev, hwtid, skb); 1534 return; 1535 } 1536 1537 static void get_4tuple(struct cpl_pass_accept_req *req, 1538 __be32 *local_ip, __be32 *peer_ip, 1539 __be16 *local_port, __be16 *peer_port) 1540 { 1541 int eth_len = G_ETH_HDR_LEN(be32_to_cpu(req->hdr_len)); 1542 int ip_len = G_IP_HDR_LEN(be32_to_cpu(req->hdr_len)); 1543 struct iphdr *ip = (struct iphdr *)((u8 *)(req + 1) + eth_len); 1544 struct tcphdr *tcp = (struct tcphdr *) 1545 ((u8 *)(req + 1) + eth_len + ip_len); 1546 1547 PDBG("%s saddr 0x%x daddr 0x%x sport %u dport %u\n", __func__, 1548 ntohl(ip->saddr), ntohl(ip->daddr), ntohs(tcp->source), 1549 ntohs(tcp->dest)); 1550 1551 *peer_ip = ip->saddr; 1552 *local_ip = ip->daddr; 1553 *peer_port = tcp->source; 1554 *local_port = tcp->dest; 1555 1556 return; 1557 } 1558 1559 static int import_ep(struct c4iw_ep *ep, __be32 peer_ip, struct dst_entry *dst, 1560 struct c4iw_dev *cdev, bool clear_mpa_v1) 1561 { 1562 struct neighbour *n; 1563 int err, step; 1564 1565 n = dst_neigh_lookup(dst, &peer_ip); 1566 if (!n) 1567 return -ENODEV; 1568 1569 rcu_read_lock(); 1570 err = -ENOMEM; 1571 if (n->dev->flags & IFF_LOOPBACK) { 1572 struct net_device *pdev; 1573 1574 pdev = ip_dev_find(&init_net, peer_ip); 1575 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1576 n, pdev, 0); 1577 if (!ep->l2t) 1578 goto out; 1579 ep->mtu = pdev->mtu; 1580 ep->tx_chan = cxgb4_port_chan(pdev); 1581 ep->smac_idx = (cxgb4_port_viid(pdev) & 0x7F) << 1; 1582 step = cdev->rdev.lldi.ntxq / 1583 cdev->rdev.lldi.nchan; 1584 ep->txq_idx = cxgb4_port_idx(pdev) * step; 1585 step = cdev->rdev.lldi.nrxq / 1586 cdev->rdev.lldi.nchan; 1587 ep->ctrlq_idx = cxgb4_port_idx(pdev); 1588 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1589 cxgb4_port_idx(pdev) * step]; 1590 dev_put(pdev); 1591 } else { 1592 ep->l2t = cxgb4_l2t_get(cdev->rdev.lldi.l2t, 1593 n, n->dev, 0); 1594 if (!ep->l2t) 1595 goto out; 1596 ep->mtu = dst_mtu(ep->dst); 1597 ep->tx_chan = cxgb4_port_chan(n->dev); 1598 ep->smac_idx = (cxgb4_port_viid(n->dev) & 0x7F) << 1; 1599 step = cdev->rdev.lldi.ntxq / 1600 cdev->rdev.lldi.nchan; 1601 ep->txq_idx = cxgb4_port_idx(n->dev) * step; 1602 ep->ctrlq_idx = cxgb4_port_idx(n->dev); 1603 step = cdev->rdev.lldi.nrxq / 1604 cdev->rdev.lldi.nchan; 1605 ep->rss_qid = cdev->rdev.lldi.rxq_ids[ 1606 cxgb4_port_idx(n->dev) * step]; 1607 1608 if (clear_mpa_v1) { 1609 ep->retry_with_mpa_v1 = 0; 1610 ep->tried_with_mpa_v1 = 0; 1611 } 1612 } 1613 err = 0; 1614 out: 1615 rcu_read_unlock(); 1616 1617 neigh_release(n); 1618 1619 return err; 1620 } 1621 1622 static int pass_accept_req(struct c4iw_dev *dev, struct sk_buff *skb) 1623 { 1624 struct c4iw_ep *child_ep, *parent_ep; 1625 struct cpl_pass_accept_req *req = cplhdr(skb); 1626 unsigned int stid = GET_POPEN_TID(ntohl(req->tos_stid)); 1627 struct tid_info *t = dev->rdev.lldi.tids; 1628 unsigned int hwtid = GET_TID(req); 1629 struct dst_entry *dst; 1630 struct rtable *rt; 1631 __be32 local_ip, peer_ip; 1632 __be16 local_port, peer_port; 1633 int err; 1634 1635 parent_ep = lookup_stid(t, stid); 1636 PDBG("%s parent ep %p tid %u\n", __func__, parent_ep, hwtid); 1637 1638 get_4tuple(req, &local_ip, &peer_ip, &local_port, &peer_port); 1639 1640 if (state_read(&parent_ep->com) != LISTEN) { 1641 printk(KERN_ERR "%s - listening ep not in LISTEN\n", 1642 __func__); 1643 goto reject; 1644 } 1645 1646 /* Find output route */ 1647 rt = find_route(dev, local_ip, peer_ip, local_port, peer_port, 1648 GET_POPEN_TOS(ntohl(req->tos_stid))); 1649 if (!rt) { 1650 printk(KERN_ERR MOD "%s - failed to find dst entry!\n", 1651 __func__); 1652 goto reject; 1653 } 1654 dst = &rt->dst; 1655 1656 child_ep = alloc_ep(sizeof(*child_ep), GFP_KERNEL); 1657 if (!child_ep) { 1658 printk(KERN_ERR MOD "%s - failed to allocate ep entry!\n", 1659 __func__); 1660 dst_release(dst); 1661 goto reject; 1662 } 1663 1664 err = import_ep(child_ep, peer_ip, dst, dev, false); 1665 if (err) { 1666 printk(KERN_ERR MOD "%s - failed to allocate l2t entry!\n", 1667 __func__); 1668 dst_release(dst); 1669 kfree(child_ep); 1670 goto reject; 1671 } 1672 1673 state_set(&child_ep->com, CONNECTING); 1674 child_ep->com.dev = dev; 1675 child_ep->com.cm_id = NULL; 1676 child_ep->com.local_addr.sin_family = PF_INET; 1677 child_ep->com.local_addr.sin_port = local_port; 1678 child_ep->com.local_addr.sin_addr.s_addr = local_ip; 1679 child_ep->com.remote_addr.sin_family = PF_INET; 1680 child_ep->com.remote_addr.sin_port = peer_port; 1681 child_ep->com.remote_addr.sin_addr.s_addr = peer_ip; 1682 c4iw_get_ep(&parent_ep->com); 1683 child_ep->parent_ep = parent_ep; 1684 child_ep->tos = GET_POPEN_TOS(ntohl(req->tos_stid)); 1685 child_ep->dst = dst; 1686 child_ep->hwtid = hwtid; 1687 1688 PDBG("%s tx_chan %u smac_idx %u rss_qid %u\n", __func__, 1689 child_ep->tx_chan, child_ep->smac_idx, child_ep->rss_qid); 1690 1691 init_timer(&child_ep->timer); 1692 cxgb4_insert_tid(t, child_ep, hwtid); 1693 accept_cr(child_ep, peer_ip, skb, req); 1694 goto out; 1695 reject: 1696 reject_cr(dev, hwtid, peer_ip, skb); 1697 out: 1698 return 0; 1699 } 1700 1701 static int pass_establish(struct c4iw_dev *dev, struct sk_buff *skb) 1702 { 1703 struct c4iw_ep *ep; 1704 struct cpl_pass_establish *req = cplhdr(skb); 1705 struct tid_info *t = dev->rdev.lldi.tids; 1706 unsigned int tid = GET_TID(req); 1707 1708 ep = lookup_tid(t, tid); 1709 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1710 ep->snd_seq = be32_to_cpu(req->snd_isn); 1711 ep->rcv_seq = be32_to_cpu(req->rcv_isn); 1712 1713 set_emss(ep, ntohs(req->tcp_opt)); 1714 1715 dst_confirm(ep->dst); 1716 state_set(&ep->com, MPA_REQ_WAIT); 1717 start_ep_timer(ep); 1718 send_flowc(ep, skb); 1719 1720 return 0; 1721 } 1722 1723 static int peer_close(struct c4iw_dev *dev, struct sk_buff *skb) 1724 { 1725 struct cpl_peer_close *hdr = cplhdr(skb); 1726 struct c4iw_ep *ep; 1727 struct c4iw_qp_attributes attrs; 1728 int disconnect = 1; 1729 int release = 0; 1730 struct tid_info *t = dev->rdev.lldi.tids; 1731 unsigned int tid = GET_TID(hdr); 1732 int ret; 1733 1734 ep = lookup_tid(t, tid); 1735 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 1736 dst_confirm(ep->dst); 1737 1738 mutex_lock(&ep->com.mutex); 1739 switch (ep->com.state) { 1740 case MPA_REQ_WAIT: 1741 __state_set(&ep->com, CLOSING); 1742 break; 1743 case MPA_REQ_SENT: 1744 __state_set(&ep->com, CLOSING); 1745 connect_reply_upcall(ep, -ECONNRESET); 1746 break; 1747 case MPA_REQ_RCVD: 1748 1749 /* 1750 * We're gonna mark this puppy DEAD, but keep 1751 * the reference on it until the ULP accepts or 1752 * rejects the CR. Also wake up anyone waiting 1753 * in rdma connection migration (see c4iw_accept_cr()). 1754 */ 1755 __state_set(&ep->com, CLOSING); 1756 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 1757 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1758 break; 1759 case MPA_REP_SENT: 1760 __state_set(&ep->com, CLOSING); 1761 PDBG("waking up ep %p tid %u\n", ep, ep->hwtid); 1762 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1763 break; 1764 case FPDU_MODE: 1765 start_ep_timer(ep); 1766 __state_set(&ep->com, CLOSING); 1767 attrs.next_state = C4IW_QP_STATE_CLOSING; 1768 ret = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1769 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1770 if (ret != -ECONNRESET) { 1771 peer_close_upcall(ep); 1772 disconnect = 1; 1773 } 1774 break; 1775 case ABORTING: 1776 disconnect = 0; 1777 break; 1778 case CLOSING: 1779 __state_set(&ep->com, MORIBUND); 1780 disconnect = 0; 1781 break; 1782 case MORIBUND: 1783 stop_ep_timer(ep); 1784 if (ep->com.cm_id && ep->com.qp) { 1785 attrs.next_state = C4IW_QP_STATE_IDLE; 1786 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 1787 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 1788 } 1789 close_complete_upcall(ep); 1790 __state_set(&ep->com, DEAD); 1791 release = 1; 1792 disconnect = 0; 1793 break; 1794 case DEAD: 1795 disconnect = 0; 1796 break; 1797 default: 1798 BUG_ON(1); 1799 } 1800 mutex_unlock(&ep->com.mutex); 1801 if (disconnect) 1802 c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 1803 if (release) 1804 release_ep_resources(ep); 1805 return 0; 1806 } 1807 1808 /* 1809 * Returns whether an ABORT_REQ_RSS message is a negative advice. 1810 */ 1811 static int is_neg_adv_abort(unsigned int status) 1812 { 1813 return status == CPL_ERR_RTX_NEG_ADVICE || 1814 status == CPL_ERR_PERSIST_NEG_ADVICE; 1815 } 1816 1817 static int c4iw_reconnect(struct c4iw_ep *ep) 1818 { 1819 struct rtable *rt; 1820 int err = 0; 1821 1822 PDBG("%s qp %p cm_id %p\n", __func__, ep->com.qp, ep->com.cm_id); 1823 init_timer(&ep->timer); 1824 1825 /* 1826 * Allocate an active TID to initiate a TCP connection. 1827 */ 1828 ep->atid = cxgb4_alloc_atid(ep->com.dev->rdev.lldi.tids, ep); 1829 if (ep->atid == -1) { 1830 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 1831 err = -ENOMEM; 1832 goto fail2; 1833 } 1834 1835 /* find a route */ 1836 rt = find_route(ep->com.dev, 1837 ep->com.cm_id->local_addr.sin_addr.s_addr, 1838 ep->com.cm_id->remote_addr.sin_addr.s_addr, 1839 ep->com.cm_id->local_addr.sin_port, 1840 ep->com.cm_id->remote_addr.sin_port, 0); 1841 if (!rt) { 1842 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 1843 err = -EHOSTUNREACH; 1844 goto fail3; 1845 } 1846 ep->dst = &rt->dst; 1847 1848 err = import_ep(ep, ep->com.cm_id->remote_addr.sin_addr.s_addr, 1849 ep->dst, ep->com.dev, false); 1850 if (err) { 1851 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 1852 goto fail4; 1853 } 1854 1855 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 1856 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 1857 ep->l2t->idx); 1858 1859 state_set(&ep->com, CONNECTING); 1860 ep->tos = 0; 1861 1862 /* send connect request to rnic */ 1863 err = send_connect(ep); 1864 if (!err) 1865 goto out; 1866 1867 cxgb4_l2t_release(ep->l2t); 1868 fail4: 1869 dst_release(ep->dst); 1870 fail3: 1871 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 1872 fail2: 1873 /* 1874 * remember to send notification to upper layer. 1875 * We are in here so the upper layer is not aware that this is 1876 * re-connect attempt and so, upper layer is still waiting for 1877 * response of 1st connect request. 1878 */ 1879 connect_reply_upcall(ep, -ECONNRESET); 1880 c4iw_put_ep(&ep->com); 1881 out: 1882 return err; 1883 } 1884 1885 static int peer_abort(struct c4iw_dev *dev, struct sk_buff *skb) 1886 { 1887 struct cpl_abort_req_rss *req = cplhdr(skb); 1888 struct c4iw_ep *ep; 1889 struct cpl_abort_rpl *rpl; 1890 struct sk_buff *rpl_skb; 1891 struct c4iw_qp_attributes attrs; 1892 int ret; 1893 int release = 0; 1894 struct tid_info *t = dev->rdev.lldi.tids; 1895 unsigned int tid = GET_TID(req); 1896 1897 ep = lookup_tid(t, tid); 1898 if (is_neg_adv_abort(req->status)) { 1899 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 1900 ep->hwtid); 1901 return 0; 1902 } 1903 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 1904 ep->com.state); 1905 1906 /* 1907 * Wake up any threads in rdma_init() or rdma_fini(). 1908 * However, this is not needed if com state is just 1909 * MPA_REQ_SENT 1910 */ 1911 if (ep->com.state != MPA_REQ_SENT) 1912 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 1913 1914 mutex_lock(&ep->com.mutex); 1915 switch (ep->com.state) { 1916 case CONNECTING: 1917 break; 1918 case MPA_REQ_WAIT: 1919 stop_ep_timer(ep); 1920 break; 1921 case MPA_REQ_SENT: 1922 stop_ep_timer(ep); 1923 if (mpa_rev == 2 && ep->tried_with_mpa_v1) 1924 connect_reply_upcall(ep, -ECONNRESET); 1925 else { 1926 /* 1927 * we just don't send notification upwards because we 1928 * want to retry with mpa_v1 without upper layers even 1929 * knowing it. 1930 * 1931 * do some housekeeping so as to re-initiate the 1932 * connection 1933 */ 1934 PDBG("%s: mpa_rev=%d. Retrying with mpav1\n", __func__, 1935 mpa_rev); 1936 ep->retry_with_mpa_v1 = 1; 1937 } 1938 break; 1939 case MPA_REP_SENT: 1940 break; 1941 case MPA_REQ_RCVD: 1942 break; 1943 case MORIBUND: 1944 case CLOSING: 1945 stop_ep_timer(ep); 1946 /*FALLTHROUGH*/ 1947 case FPDU_MODE: 1948 if (ep->com.cm_id && ep->com.qp) { 1949 attrs.next_state = C4IW_QP_STATE_ERROR; 1950 ret = c4iw_modify_qp(ep->com.qp->rhp, 1951 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 1952 &attrs, 1); 1953 if (ret) 1954 printk(KERN_ERR MOD 1955 "%s - qp <- error failed!\n", 1956 __func__); 1957 } 1958 peer_abort_upcall(ep); 1959 break; 1960 case ABORTING: 1961 break; 1962 case DEAD: 1963 PDBG("%s PEER_ABORT IN DEAD STATE!!!!\n", __func__); 1964 mutex_unlock(&ep->com.mutex); 1965 return 0; 1966 default: 1967 BUG_ON(1); 1968 break; 1969 } 1970 dst_confirm(ep->dst); 1971 if (ep->com.state != ABORTING) { 1972 __state_set(&ep->com, DEAD); 1973 /* we don't release if we want to retry with mpa_v1 */ 1974 if (!ep->retry_with_mpa_v1) 1975 release = 1; 1976 } 1977 mutex_unlock(&ep->com.mutex); 1978 1979 rpl_skb = get_skb(skb, sizeof(*rpl), GFP_KERNEL); 1980 if (!rpl_skb) { 1981 printk(KERN_ERR MOD "%s - cannot allocate skb!\n", 1982 __func__); 1983 release = 1; 1984 goto out; 1985 } 1986 set_wr_txq(skb, CPL_PRIORITY_DATA, ep->txq_idx); 1987 rpl = (struct cpl_abort_rpl *) skb_put(rpl_skb, sizeof(*rpl)); 1988 INIT_TP_WR(rpl, ep->hwtid); 1989 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, ep->hwtid)); 1990 rpl->cmd = CPL_ABORT_NO_RST; 1991 c4iw_ofld_send(&ep->com.dev->rdev, rpl_skb); 1992 out: 1993 if (release) 1994 release_ep_resources(ep); 1995 1996 /* retry with mpa-v1 */ 1997 if (ep && ep->retry_with_mpa_v1) { 1998 cxgb4_remove_tid(ep->com.dev->rdev.lldi.tids, 0, ep->hwtid); 1999 dst_release(ep->dst); 2000 cxgb4_l2t_release(ep->l2t); 2001 c4iw_reconnect(ep); 2002 } 2003 2004 return 0; 2005 } 2006 2007 static int close_con_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2008 { 2009 struct c4iw_ep *ep; 2010 struct c4iw_qp_attributes attrs; 2011 struct cpl_close_con_rpl *rpl = cplhdr(skb); 2012 int release = 0; 2013 struct tid_info *t = dev->rdev.lldi.tids; 2014 unsigned int tid = GET_TID(rpl); 2015 2016 ep = lookup_tid(t, tid); 2017 2018 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2019 BUG_ON(!ep); 2020 2021 /* The cm_id may be null if we failed to connect */ 2022 mutex_lock(&ep->com.mutex); 2023 switch (ep->com.state) { 2024 case CLOSING: 2025 __state_set(&ep->com, MORIBUND); 2026 break; 2027 case MORIBUND: 2028 stop_ep_timer(ep); 2029 if ((ep->com.cm_id) && (ep->com.qp)) { 2030 attrs.next_state = C4IW_QP_STATE_IDLE; 2031 c4iw_modify_qp(ep->com.qp->rhp, 2032 ep->com.qp, 2033 C4IW_QP_ATTR_NEXT_STATE, 2034 &attrs, 1); 2035 } 2036 close_complete_upcall(ep); 2037 __state_set(&ep->com, DEAD); 2038 release = 1; 2039 break; 2040 case ABORTING: 2041 case DEAD: 2042 break; 2043 default: 2044 BUG_ON(1); 2045 break; 2046 } 2047 mutex_unlock(&ep->com.mutex); 2048 if (release) 2049 release_ep_resources(ep); 2050 return 0; 2051 } 2052 2053 static int terminate(struct c4iw_dev *dev, struct sk_buff *skb) 2054 { 2055 struct cpl_rdma_terminate *rpl = cplhdr(skb); 2056 struct tid_info *t = dev->rdev.lldi.tids; 2057 unsigned int tid = GET_TID(rpl); 2058 struct c4iw_ep *ep; 2059 struct c4iw_qp_attributes attrs; 2060 2061 ep = lookup_tid(t, tid); 2062 BUG_ON(!ep); 2063 2064 if (ep && ep->com.qp) { 2065 printk(KERN_WARNING MOD "TERM received tid %u qpid %u\n", tid, 2066 ep->com.qp->wq.sq.qid); 2067 attrs.next_state = C4IW_QP_STATE_TERMINATE; 2068 c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, 2069 C4IW_QP_ATTR_NEXT_STATE, &attrs, 1); 2070 } else 2071 printk(KERN_WARNING MOD "TERM received tid %u no ep/qp\n", tid); 2072 2073 return 0; 2074 } 2075 2076 /* 2077 * Upcall from the adapter indicating data has been transmitted. 2078 * For us its just the single MPA request or reply. We can now free 2079 * the skb holding the mpa message. 2080 */ 2081 static int fw4_ack(struct c4iw_dev *dev, struct sk_buff *skb) 2082 { 2083 struct c4iw_ep *ep; 2084 struct cpl_fw4_ack *hdr = cplhdr(skb); 2085 u8 credits = hdr->credits; 2086 unsigned int tid = GET_TID(hdr); 2087 struct tid_info *t = dev->rdev.lldi.tids; 2088 2089 2090 ep = lookup_tid(t, tid); 2091 PDBG("%s ep %p tid %u credits %u\n", __func__, ep, ep->hwtid, credits); 2092 if (credits == 0) { 2093 PDBG("%s 0 credit ack ep %p tid %u state %u\n", 2094 __func__, ep, ep->hwtid, state_read(&ep->com)); 2095 return 0; 2096 } 2097 2098 dst_confirm(ep->dst); 2099 if (ep->mpa_skb) { 2100 PDBG("%s last streaming msg ack ep %p tid %u state %u " 2101 "initiator %u freeing skb\n", __func__, ep, ep->hwtid, 2102 state_read(&ep->com), ep->mpa_attr.initiator ? 1 : 0); 2103 kfree_skb(ep->mpa_skb); 2104 ep->mpa_skb = NULL; 2105 } 2106 return 0; 2107 } 2108 2109 int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) 2110 { 2111 int err; 2112 struct c4iw_ep *ep = to_ep(cm_id); 2113 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2114 2115 if (state_read(&ep->com) == DEAD) { 2116 c4iw_put_ep(&ep->com); 2117 return -ECONNRESET; 2118 } 2119 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2120 if (mpa_rev == 0) 2121 abort_connection(ep, NULL, GFP_KERNEL); 2122 else { 2123 err = send_mpa_reject(ep, pdata, pdata_len); 2124 err = c4iw_ep_disconnect(ep, 0, GFP_KERNEL); 2125 } 2126 c4iw_put_ep(&ep->com); 2127 return 0; 2128 } 2129 2130 int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2131 { 2132 int err; 2133 struct c4iw_qp_attributes attrs; 2134 enum c4iw_qp_attr_mask mask; 2135 struct c4iw_ep *ep = to_ep(cm_id); 2136 struct c4iw_dev *h = to_c4iw_dev(cm_id->device); 2137 struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); 2138 2139 PDBG("%s ep %p tid %u\n", __func__, ep, ep->hwtid); 2140 if (state_read(&ep->com) == DEAD) { 2141 err = -ECONNRESET; 2142 goto err; 2143 } 2144 2145 BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); 2146 BUG_ON(!qp); 2147 2148 if ((conn_param->ord > c4iw_max_read_depth) || 2149 (conn_param->ird > c4iw_max_read_depth)) { 2150 abort_connection(ep, NULL, GFP_KERNEL); 2151 err = -EINVAL; 2152 goto err; 2153 } 2154 2155 if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { 2156 if (conn_param->ord > ep->ird) { 2157 ep->ird = conn_param->ird; 2158 ep->ord = conn_param->ord; 2159 send_mpa_reject(ep, conn_param->private_data, 2160 conn_param->private_data_len); 2161 abort_connection(ep, NULL, GFP_KERNEL); 2162 err = -ENOMEM; 2163 goto err; 2164 } 2165 if (conn_param->ird > ep->ord) { 2166 if (!ep->ord) 2167 conn_param->ird = 1; 2168 else { 2169 abort_connection(ep, NULL, GFP_KERNEL); 2170 err = -ENOMEM; 2171 goto err; 2172 } 2173 } 2174 2175 } 2176 ep->ird = conn_param->ird; 2177 ep->ord = conn_param->ord; 2178 2179 if (ep->mpa_attr.version != 2) 2180 if (peer2peer && ep->ird == 0) 2181 ep->ird = 1; 2182 2183 PDBG("%s %d ird %d ord %d\n", __func__, __LINE__, ep->ird, ep->ord); 2184 2185 cm_id->add_ref(cm_id); 2186 ep->com.cm_id = cm_id; 2187 ep->com.qp = qp; 2188 2189 /* bind QP to EP and move to RTS */ 2190 attrs.mpa_attr = ep->mpa_attr; 2191 attrs.max_ird = ep->ird; 2192 attrs.max_ord = ep->ord; 2193 attrs.llp_stream_handle = ep; 2194 attrs.next_state = C4IW_QP_STATE_RTS; 2195 2196 /* bind QP and TID with INIT_WR */ 2197 mask = C4IW_QP_ATTR_NEXT_STATE | 2198 C4IW_QP_ATTR_LLP_STREAM_HANDLE | 2199 C4IW_QP_ATTR_MPA_ATTR | 2200 C4IW_QP_ATTR_MAX_IRD | 2201 C4IW_QP_ATTR_MAX_ORD; 2202 2203 err = c4iw_modify_qp(ep->com.qp->rhp, 2204 ep->com.qp, mask, &attrs, 1); 2205 if (err) 2206 goto err1; 2207 err = send_mpa_reply(ep, conn_param->private_data, 2208 conn_param->private_data_len); 2209 if (err) 2210 goto err1; 2211 2212 state_set(&ep->com, FPDU_MODE); 2213 established_upcall(ep); 2214 c4iw_put_ep(&ep->com); 2215 return 0; 2216 err1: 2217 ep->com.cm_id = NULL; 2218 ep->com.qp = NULL; 2219 cm_id->rem_ref(cm_id); 2220 err: 2221 c4iw_put_ep(&ep->com); 2222 return err; 2223 } 2224 2225 int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) 2226 { 2227 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2228 struct c4iw_ep *ep; 2229 struct rtable *rt; 2230 int err = 0; 2231 2232 if ((conn_param->ord > c4iw_max_read_depth) || 2233 (conn_param->ird > c4iw_max_read_depth)) { 2234 err = -EINVAL; 2235 goto out; 2236 } 2237 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2238 if (!ep) { 2239 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2240 err = -ENOMEM; 2241 goto out; 2242 } 2243 init_timer(&ep->timer); 2244 ep->plen = conn_param->private_data_len; 2245 if (ep->plen) 2246 memcpy(ep->mpa_pkt + sizeof(struct mpa_message), 2247 conn_param->private_data, ep->plen); 2248 ep->ird = conn_param->ird; 2249 ep->ord = conn_param->ord; 2250 2251 if (peer2peer && ep->ord == 0) 2252 ep->ord = 1; 2253 2254 cm_id->add_ref(cm_id); 2255 ep->com.dev = dev; 2256 ep->com.cm_id = cm_id; 2257 ep->com.qp = get_qhp(dev, conn_param->qpn); 2258 BUG_ON(!ep->com.qp); 2259 PDBG("%s qpn 0x%x qp %p cm_id %p\n", __func__, conn_param->qpn, 2260 ep->com.qp, cm_id); 2261 2262 /* 2263 * Allocate an active TID to initiate a TCP connection. 2264 */ 2265 ep->atid = cxgb4_alloc_atid(dev->rdev.lldi.tids, ep); 2266 if (ep->atid == -1) { 2267 printk(KERN_ERR MOD "%s - cannot alloc atid.\n", __func__); 2268 err = -ENOMEM; 2269 goto fail2; 2270 } 2271 2272 PDBG("%s saddr 0x%x sport 0x%x raddr 0x%x rport 0x%x\n", __func__, 2273 ntohl(cm_id->local_addr.sin_addr.s_addr), 2274 ntohs(cm_id->local_addr.sin_port), 2275 ntohl(cm_id->remote_addr.sin_addr.s_addr), 2276 ntohs(cm_id->remote_addr.sin_port)); 2277 2278 /* find a route */ 2279 rt = find_route(dev, 2280 cm_id->local_addr.sin_addr.s_addr, 2281 cm_id->remote_addr.sin_addr.s_addr, 2282 cm_id->local_addr.sin_port, 2283 cm_id->remote_addr.sin_port, 0); 2284 if (!rt) { 2285 printk(KERN_ERR MOD "%s - cannot find route.\n", __func__); 2286 err = -EHOSTUNREACH; 2287 goto fail3; 2288 } 2289 ep->dst = &rt->dst; 2290 2291 err = import_ep(ep, cm_id->remote_addr.sin_addr.s_addr, 2292 ep->dst, ep->com.dev, true); 2293 if (err) { 2294 printk(KERN_ERR MOD "%s - cannot alloc l2e.\n", __func__); 2295 goto fail4; 2296 } 2297 2298 PDBG("%s txq_idx %u tx_chan %u smac_idx %u rss_qid %u l2t_idx %u\n", 2299 __func__, ep->txq_idx, ep->tx_chan, ep->smac_idx, ep->rss_qid, 2300 ep->l2t->idx); 2301 2302 state_set(&ep->com, CONNECTING); 2303 ep->tos = 0; 2304 ep->com.local_addr = cm_id->local_addr; 2305 ep->com.remote_addr = cm_id->remote_addr; 2306 2307 /* send connect request to rnic */ 2308 err = send_connect(ep); 2309 if (!err) 2310 goto out; 2311 2312 cxgb4_l2t_release(ep->l2t); 2313 fail4: 2314 dst_release(ep->dst); 2315 fail3: 2316 cxgb4_free_atid(ep->com.dev->rdev.lldi.tids, ep->atid); 2317 fail2: 2318 cm_id->rem_ref(cm_id); 2319 c4iw_put_ep(&ep->com); 2320 out: 2321 return err; 2322 } 2323 2324 int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog) 2325 { 2326 int err = 0; 2327 struct c4iw_dev *dev = to_c4iw_dev(cm_id->device); 2328 struct c4iw_listen_ep *ep; 2329 2330 2331 might_sleep(); 2332 2333 ep = alloc_ep(sizeof(*ep), GFP_KERNEL); 2334 if (!ep) { 2335 printk(KERN_ERR MOD "%s - cannot alloc ep.\n", __func__); 2336 err = -ENOMEM; 2337 goto fail1; 2338 } 2339 PDBG("%s ep %p\n", __func__, ep); 2340 cm_id->add_ref(cm_id); 2341 ep->com.cm_id = cm_id; 2342 ep->com.dev = dev; 2343 ep->backlog = backlog; 2344 ep->com.local_addr = cm_id->local_addr; 2345 2346 /* 2347 * Allocate a server TID. 2348 */ 2349 ep->stid = cxgb4_alloc_stid(dev->rdev.lldi.tids, PF_INET, ep); 2350 if (ep->stid == -1) { 2351 printk(KERN_ERR MOD "%s - cannot alloc stid.\n", __func__); 2352 err = -ENOMEM; 2353 goto fail2; 2354 } 2355 2356 state_set(&ep->com, LISTEN); 2357 c4iw_init_wr_wait(&ep->com.wr_wait); 2358 err = cxgb4_create_server(ep->com.dev->rdev.lldi.ports[0], ep->stid, 2359 ep->com.local_addr.sin_addr.s_addr, 2360 ep->com.local_addr.sin_port, 2361 ep->com.dev->rdev.lldi.rxq_ids[0]); 2362 if (err) 2363 goto fail3; 2364 2365 /* wait for pass_open_rpl */ 2366 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 0, 0, 2367 __func__); 2368 if (!err) { 2369 cm_id->provider_data = ep; 2370 goto out; 2371 } 2372 fail3: 2373 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2374 fail2: 2375 cm_id->rem_ref(cm_id); 2376 c4iw_put_ep(&ep->com); 2377 fail1: 2378 out: 2379 return err; 2380 } 2381 2382 int c4iw_destroy_listen(struct iw_cm_id *cm_id) 2383 { 2384 int err; 2385 struct c4iw_listen_ep *ep = to_listen_ep(cm_id); 2386 2387 PDBG("%s ep %p\n", __func__, ep); 2388 2389 might_sleep(); 2390 state_set(&ep->com, DEAD); 2391 c4iw_init_wr_wait(&ep->com.wr_wait); 2392 err = listen_stop(ep); 2393 if (err) 2394 goto done; 2395 err = c4iw_wait_for_reply(&ep->com.dev->rdev, &ep->com.wr_wait, 0, 0, 2396 __func__); 2397 cxgb4_free_stid(ep->com.dev->rdev.lldi.tids, ep->stid, PF_INET); 2398 done: 2399 cm_id->rem_ref(cm_id); 2400 c4iw_put_ep(&ep->com); 2401 return err; 2402 } 2403 2404 int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) 2405 { 2406 int ret = 0; 2407 int close = 0; 2408 int fatal = 0; 2409 struct c4iw_rdev *rdev; 2410 2411 mutex_lock(&ep->com.mutex); 2412 2413 PDBG("%s ep %p state %s, abrupt %d\n", __func__, ep, 2414 states[ep->com.state], abrupt); 2415 2416 rdev = &ep->com.dev->rdev; 2417 if (c4iw_fatal_error(rdev)) { 2418 fatal = 1; 2419 close_complete_upcall(ep); 2420 ep->com.state = DEAD; 2421 } 2422 switch (ep->com.state) { 2423 case MPA_REQ_WAIT: 2424 case MPA_REQ_SENT: 2425 case MPA_REQ_RCVD: 2426 case MPA_REP_SENT: 2427 case FPDU_MODE: 2428 close = 1; 2429 if (abrupt) 2430 ep->com.state = ABORTING; 2431 else { 2432 ep->com.state = CLOSING; 2433 start_ep_timer(ep); 2434 } 2435 set_bit(CLOSE_SENT, &ep->com.flags); 2436 break; 2437 case CLOSING: 2438 if (!test_and_set_bit(CLOSE_SENT, &ep->com.flags)) { 2439 close = 1; 2440 if (abrupt) { 2441 stop_ep_timer(ep); 2442 ep->com.state = ABORTING; 2443 } else 2444 ep->com.state = MORIBUND; 2445 } 2446 break; 2447 case MORIBUND: 2448 case ABORTING: 2449 case DEAD: 2450 PDBG("%s ignoring disconnect ep %p state %u\n", 2451 __func__, ep, ep->com.state); 2452 break; 2453 default: 2454 BUG(); 2455 break; 2456 } 2457 2458 if (close) { 2459 if (abrupt) { 2460 close_complete_upcall(ep); 2461 ret = send_abort(ep, NULL, gfp); 2462 } else 2463 ret = send_halfclose(ep, gfp); 2464 if (ret) 2465 fatal = 1; 2466 } 2467 mutex_unlock(&ep->com.mutex); 2468 if (fatal) 2469 release_ep_resources(ep); 2470 return ret; 2471 } 2472 2473 static int async_event(struct c4iw_dev *dev, struct sk_buff *skb) 2474 { 2475 struct cpl_fw6_msg *rpl = cplhdr(skb); 2476 c4iw_ev_dispatch(dev, (struct t4_cqe *)&rpl->data[0]); 2477 return 0; 2478 } 2479 2480 /* 2481 * These are the real handlers that are called from a 2482 * work queue. 2483 */ 2484 static c4iw_handler_func work_handlers[NUM_CPL_CMDS] = { 2485 [CPL_ACT_ESTABLISH] = act_establish, 2486 [CPL_ACT_OPEN_RPL] = act_open_rpl, 2487 [CPL_RX_DATA] = rx_data, 2488 [CPL_ABORT_RPL_RSS] = abort_rpl, 2489 [CPL_ABORT_RPL] = abort_rpl, 2490 [CPL_PASS_OPEN_RPL] = pass_open_rpl, 2491 [CPL_CLOSE_LISTSRV_RPL] = close_listsrv_rpl, 2492 [CPL_PASS_ACCEPT_REQ] = pass_accept_req, 2493 [CPL_PASS_ESTABLISH] = pass_establish, 2494 [CPL_PEER_CLOSE] = peer_close, 2495 [CPL_ABORT_REQ_RSS] = peer_abort, 2496 [CPL_CLOSE_CON_RPL] = close_con_rpl, 2497 [CPL_RDMA_TERMINATE] = terminate, 2498 [CPL_FW4_ACK] = fw4_ack, 2499 [CPL_FW6_MSG] = async_event 2500 }; 2501 2502 static void process_timeout(struct c4iw_ep *ep) 2503 { 2504 struct c4iw_qp_attributes attrs; 2505 int abort = 1; 2506 2507 mutex_lock(&ep->com.mutex); 2508 PDBG("%s ep %p tid %u state %d\n", __func__, ep, ep->hwtid, 2509 ep->com.state); 2510 switch (ep->com.state) { 2511 case MPA_REQ_SENT: 2512 __state_set(&ep->com, ABORTING); 2513 connect_reply_upcall(ep, -ETIMEDOUT); 2514 break; 2515 case MPA_REQ_WAIT: 2516 __state_set(&ep->com, ABORTING); 2517 break; 2518 case CLOSING: 2519 case MORIBUND: 2520 if (ep->com.cm_id && ep->com.qp) { 2521 attrs.next_state = C4IW_QP_STATE_ERROR; 2522 c4iw_modify_qp(ep->com.qp->rhp, 2523 ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, 2524 &attrs, 1); 2525 } 2526 __state_set(&ep->com, ABORTING); 2527 break; 2528 default: 2529 printk(KERN_ERR "%s unexpected state ep %p tid %u state %u\n", 2530 __func__, ep, ep->hwtid, ep->com.state); 2531 WARN_ON(1); 2532 abort = 0; 2533 } 2534 mutex_unlock(&ep->com.mutex); 2535 if (abort) 2536 abort_connection(ep, NULL, GFP_KERNEL); 2537 c4iw_put_ep(&ep->com); 2538 } 2539 2540 static void process_timedout_eps(void) 2541 { 2542 struct c4iw_ep *ep; 2543 2544 spin_lock_irq(&timeout_lock); 2545 while (!list_empty(&timeout_list)) { 2546 struct list_head *tmp; 2547 2548 tmp = timeout_list.next; 2549 list_del(tmp); 2550 spin_unlock_irq(&timeout_lock); 2551 ep = list_entry(tmp, struct c4iw_ep, entry); 2552 process_timeout(ep); 2553 spin_lock_irq(&timeout_lock); 2554 } 2555 spin_unlock_irq(&timeout_lock); 2556 } 2557 2558 static void process_work(struct work_struct *work) 2559 { 2560 struct sk_buff *skb = NULL; 2561 struct c4iw_dev *dev; 2562 struct cpl_act_establish *rpl; 2563 unsigned int opcode; 2564 int ret; 2565 2566 while ((skb = skb_dequeue(&rxq))) { 2567 rpl = cplhdr(skb); 2568 dev = *((struct c4iw_dev **) (skb->cb + sizeof(void *))); 2569 opcode = rpl->ot.opcode; 2570 2571 BUG_ON(!work_handlers[opcode]); 2572 ret = work_handlers[opcode](dev, skb); 2573 if (!ret) 2574 kfree_skb(skb); 2575 } 2576 process_timedout_eps(); 2577 } 2578 2579 static DECLARE_WORK(skb_work, process_work); 2580 2581 static void ep_timeout(unsigned long arg) 2582 { 2583 struct c4iw_ep *ep = (struct c4iw_ep *)arg; 2584 2585 spin_lock(&timeout_lock); 2586 list_add_tail(&ep->entry, &timeout_list); 2587 spin_unlock(&timeout_lock); 2588 queue_work(workq, &skb_work); 2589 } 2590 2591 /* 2592 * All the CM events are handled on a work queue to have a safe context. 2593 */ 2594 static int sched(struct c4iw_dev *dev, struct sk_buff *skb) 2595 { 2596 2597 /* 2598 * Save dev in the skb->cb area. 2599 */ 2600 *((struct c4iw_dev **) (skb->cb + sizeof(void *))) = dev; 2601 2602 /* 2603 * Queue the skb and schedule the worker thread. 2604 */ 2605 skb_queue_tail(&rxq, skb); 2606 queue_work(workq, &skb_work); 2607 return 0; 2608 } 2609 2610 static int set_tcb_rpl(struct c4iw_dev *dev, struct sk_buff *skb) 2611 { 2612 struct cpl_set_tcb_rpl *rpl = cplhdr(skb); 2613 2614 if (rpl->status != CPL_ERR_NONE) { 2615 printk(KERN_ERR MOD "Unexpected SET_TCB_RPL status %u " 2616 "for tid %u\n", rpl->status, GET_TID(rpl)); 2617 } 2618 kfree_skb(skb); 2619 return 0; 2620 } 2621 2622 static int fw6_msg(struct c4iw_dev *dev, struct sk_buff *skb) 2623 { 2624 struct cpl_fw6_msg *rpl = cplhdr(skb); 2625 struct c4iw_wr_wait *wr_waitp; 2626 int ret; 2627 2628 PDBG("%s type %u\n", __func__, rpl->type); 2629 2630 switch (rpl->type) { 2631 case 1: 2632 ret = (int)((be64_to_cpu(rpl->data[0]) >> 8) & 0xff); 2633 wr_waitp = (struct c4iw_wr_wait *)(__force unsigned long) rpl->data[1]; 2634 PDBG("%s wr_waitp %p ret %u\n", __func__, wr_waitp, ret); 2635 if (wr_waitp) 2636 c4iw_wake_up(wr_waitp, ret ? -ret : 0); 2637 kfree_skb(skb); 2638 break; 2639 case 2: 2640 sched(dev, skb); 2641 break; 2642 default: 2643 printk(KERN_ERR MOD "%s unexpected fw6 msg type %u\n", __func__, 2644 rpl->type); 2645 kfree_skb(skb); 2646 break; 2647 } 2648 return 0; 2649 } 2650 2651 static int peer_abort_intr(struct c4iw_dev *dev, struct sk_buff *skb) 2652 { 2653 struct cpl_abort_req_rss *req = cplhdr(skb); 2654 struct c4iw_ep *ep; 2655 struct tid_info *t = dev->rdev.lldi.tids; 2656 unsigned int tid = GET_TID(req); 2657 2658 ep = lookup_tid(t, tid); 2659 if (is_neg_adv_abort(req->status)) { 2660 PDBG("%s neg_adv_abort ep %p tid %u\n", __func__, ep, 2661 ep->hwtid); 2662 kfree_skb(skb); 2663 return 0; 2664 } 2665 PDBG("%s ep %p tid %u state %u\n", __func__, ep, ep->hwtid, 2666 ep->com.state); 2667 2668 /* 2669 * Wake up any threads in rdma_init() or rdma_fini(). 2670 * However, this is not needed if com state is just 2671 * MPA_REQ_SENT 2672 */ 2673 if (ep->com.state != MPA_REQ_SENT) 2674 c4iw_wake_up(&ep->com.wr_wait, -ECONNRESET); 2675 sched(dev, skb); 2676 return 0; 2677 } 2678 2679 /* 2680 * Most upcalls from the T4 Core go to sched() to 2681 * schedule the processing on a work queue. 2682 */ 2683 c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS] = { 2684 [CPL_ACT_ESTABLISH] = sched, 2685 [CPL_ACT_OPEN_RPL] = sched, 2686 [CPL_RX_DATA] = sched, 2687 [CPL_ABORT_RPL_RSS] = sched, 2688 [CPL_ABORT_RPL] = sched, 2689 [CPL_PASS_OPEN_RPL] = sched, 2690 [CPL_CLOSE_LISTSRV_RPL] = sched, 2691 [CPL_PASS_ACCEPT_REQ] = sched, 2692 [CPL_PASS_ESTABLISH] = sched, 2693 [CPL_PEER_CLOSE] = sched, 2694 [CPL_CLOSE_CON_RPL] = sched, 2695 [CPL_ABORT_REQ_RSS] = peer_abort_intr, 2696 [CPL_RDMA_TERMINATE] = sched, 2697 [CPL_FW4_ACK] = sched, 2698 [CPL_SET_TCB_RPL] = set_tcb_rpl, 2699 [CPL_FW6_MSG] = fw6_msg 2700 }; 2701 2702 int __init c4iw_cm_init(void) 2703 { 2704 spin_lock_init(&timeout_lock); 2705 skb_queue_head_init(&rxq); 2706 2707 workq = create_singlethread_workqueue("iw_cxgb4"); 2708 if (!workq) 2709 return -ENOMEM; 2710 2711 return 0; 2712 } 2713 2714 void __exit c4iw_cm_term(void) 2715 { 2716 WARN_ON(!list_empty(&timeout_list)); 2717 flush_workqueue(workq); 2718 destroy_workqueue(workq); 2719 } 2720