1 /* 2 * cxgb4i.c: Chelsio T4 iSCSI driver. 3 * 4 * Copyright (c) 2010-2015 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/kernel.h> 17 #include <linux/module.h> 18 #include <linux/moduleparam.h> 19 #include <scsi/scsi_host.h> 20 #include <net/tcp.h> 21 #include <net/dst.h> 22 #include <linux/netdevice.h> 23 #include <net/addrconf.h> 24 25 #include "t4_regs.h" 26 #include "t4_msg.h" 27 #include "cxgb4.h" 28 #include "cxgb4_uld.h" 29 #include "t4fw_api.h" 30 #include "l2t.h" 31 #include "cxgb4i.h" 32 #include "clip_tbl.h" 33 34 static unsigned int dbg_level; 35 36 #include "../libcxgbi.h" 37 38 #define DRV_MODULE_NAME "cxgb4i" 39 #define DRV_MODULE_DESC "Chelsio T4/T5 iSCSI Driver" 40 #define DRV_MODULE_VERSION "0.9.5-ko" 41 #define DRV_MODULE_RELDATE "Apr. 2015" 42 43 static char version[] = 44 DRV_MODULE_DESC " " DRV_MODULE_NAME 45 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 46 47 MODULE_AUTHOR("Chelsio Communications, Inc."); 48 MODULE_DESCRIPTION(DRV_MODULE_DESC); 49 MODULE_VERSION(DRV_MODULE_VERSION); 50 MODULE_LICENSE("GPL"); 51 52 module_param(dbg_level, uint, 0644); 53 MODULE_PARM_DESC(dbg_level, "Debug flag (default=0)"); 54 55 #define CXGB4I_DEFAULT_10G_RCV_WIN (256 * 1024) 56 static int cxgb4i_rcv_win = -1; 57 module_param(cxgb4i_rcv_win, int, 0644); 58 MODULE_PARM_DESC(cxgb4i_rcv_win, "TCP reveive window in bytes"); 59 60 #define CXGB4I_DEFAULT_10G_SND_WIN (128 * 1024) 61 static int cxgb4i_snd_win = -1; 62 module_param(cxgb4i_snd_win, int, 0644); 63 MODULE_PARM_DESC(cxgb4i_snd_win, "TCP send window in bytes"); 64 65 static int cxgb4i_rx_credit_thres = 10 * 1024; 66 module_param(cxgb4i_rx_credit_thres, int, 0644); 67 MODULE_PARM_DESC(cxgb4i_rx_credit_thres, 68 "RX credits return threshold in bytes (default=10KB)"); 69 70 static unsigned int cxgb4i_max_connect = (8 * 1024); 71 module_param(cxgb4i_max_connect, uint, 0644); 72 MODULE_PARM_DESC(cxgb4i_max_connect, "Maximum number of connections"); 73 74 static unsigned short cxgb4i_sport_base = 20000; 75 module_param(cxgb4i_sport_base, ushort, 0644); 76 MODULE_PARM_DESC(cxgb4i_sport_base, "Starting port number (default 20000)"); 77 78 typedef void (*cxgb4i_cplhandler_func)(struct cxgbi_device *, struct sk_buff *); 79 80 static void *t4_uld_add(const struct cxgb4_lld_info *); 81 static int t4_uld_rx_handler(void *, const __be64 *, const struct pkt_gl *); 82 static int t4_uld_state_change(void *, enum cxgb4_state state); 83 static inline int send_tx_flowc_wr(struct cxgbi_sock *); 84 85 static const struct cxgb4_uld_info cxgb4i_uld_info = { 86 .name = DRV_MODULE_NAME, 87 .nrxq = MAX_ULD_QSETS, 88 .ntxq = MAX_ULD_QSETS, 89 .rxq_size = 1024, 90 .lro = false, 91 .add = t4_uld_add, 92 .rx_handler = t4_uld_rx_handler, 93 .state_change = t4_uld_state_change, 94 }; 95 96 static struct scsi_host_template cxgb4i_host_template = { 97 .module = THIS_MODULE, 98 .name = DRV_MODULE_NAME, 99 .proc_name = DRV_MODULE_NAME, 100 .can_queue = CXGB4I_SCSI_HOST_QDEPTH, 101 .queuecommand = iscsi_queuecommand, 102 .change_queue_depth = scsi_change_queue_depth, 103 .sg_tablesize = SG_ALL, 104 .max_sectors = 0xFFFF, 105 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 106 .eh_abort_handler = iscsi_eh_abort, 107 .eh_device_reset_handler = iscsi_eh_device_reset, 108 .eh_target_reset_handler = iscsi_eh_recover_target, 109 .target_alloc = iscsi_target_alloc, 110 .use_clustering = DISABLE_CLUSTERING, 111 .this_id = -1, 112 .track_queue_depth = 1, 113 }; 114 115 static struct iscsi_transport cxgb4i_iscsi_transport = { 116 .owner = THIS_MODULE, 117 .name = DRV_MODULE_NAME, 118 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST | 119 CAP_DATADGST | CAP_DIGEST_OFFLOAD | 120 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, 121 .attr_is_visible = cxgbi_attr_is_visible, 122 .get_host_param = cxgbi_get_host_param, 123 .set_host_param = cxgbi_set_host_param, 124 /* session management */ 125 .create_session = cxgbi_create_session, 126 .destroy_session = cxgbi_destroy_session, 127 .get_session_param = iscsi_session_get_param, 128 /* connection management */ 129 .create_conn = cxgbi_create_conn, 130 .bind_conn = cxgbi_bind_conn, 131 .destroy_conn = iscsi_tcp_conn_teardown, 132 .start_conn = iscsi_conn_start, 133 .stop_conn = iscsi_conn_stop, 134 .get_conn_param = iscsi_conn_get_param, 135 .set_param = cxgbi_set_conn_param, 136 .get_stats = cxgbi_get_conn_stats, 137 /* pdu xmit req from user space */ 138 .send_pdu = iscsi_conn_send_pdu, 139 /* task */ 140 .init_task = iscsi_tcp_task_init, 141 .xmit_task = iscsi_tcp_task_xmit, 142 .cleanup_task = cxgbi_cleanup_task, 143 /* pdu */ 144 .alloc_pdu = cxgbi_conn_alloc_pdu, 145 .init_pdu = cxgbi_conn_init_pdu, 146 .xmit_pdu = cxgbi_conn_xmit_pdu, 147 .parse_pdu_itt = cxgbi_parse_pdu_itt, 148 /* TCP connect/disconnect */ 149 .get_ep_param = cxgbi_get_ep_param, 150 .ep_connect = cxgbi_ep_connect, 151 .ep_poll = cxgbi_ep_poll, 152 .ep_disconnect = cxgbi_ep_disconnect, 153 /* Error recovery timeout call */ 154 .session_recovery_timedout = iscsi_session_recovery_timedout, 155 }; 156 157 static struct scsi_transport_template *cxgb4i_stt; 158 159 /* 160 * CPL (Chelsio Protocol Language) defines a message passing interface between 161 * the host driver and Chelsio asic. 162 * The section below implments CPLs that related to iscsi tcp connection 163 * open/close/abort and data send/receive. 164 */ 165 166 #define RCV_BUFSIZ_MASK 0x3FFU 167 #define MAX_IMM_TX_PKT_LEN 256 168 169 static int push_tx_frames(struct cxgbi_sock *, int); 170 171 /* 172 * is_ofld_imm - check whether a packet can be sent as immediate data 173 * @skb: the packet 174 * 175 * Returns true if a packet can be sent as an offload WR with immediate 176 * data. We currently use the same limit as for Ethernet packets. 177 */ 178 static inline bool is_ofld_imm(const struct sk_buff *skb) 179 { 180 int len = skb->len; 181 182 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) 183 len += sizeof(struct fw_ofld_tx_data_wr); 184 185 return len <= MAX_IMM_TX_PKT_LEN; 186 } 187 188 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 189 struct l2t_entry *e) 190 { 191 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 192 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 193 unsigned long long opt0; 194 unsigned int opt2; 195 unsigned int qid_atid = ((unsigned int)csk->atid) | 196 (((unsigned int)csk->rss_qid) << 14); 197 198 opt0 = KEEP_ALIVE_F | 199 WND_SCALE_V(wscale) | 200 MSS_IDX_V(csk->mss_idx) | 201 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 202 TX_CHAN_V(csk->tx_chan) | 203 SMAC_SEL_V(csk->smac_idx) | 204 ULP_MODE_V(ULP_MODE_ISCSI) | 205 RCV_BUFSIZ_V(csk->rcv_win >> 10); 206 207 opt2 = RX_CHANNEL_V(0) | 208 RSS_QUEUE_VALID_F | 209 RSS_QUEUE_V(csk->rss_qid); 210 211 if (is_t4(lldi->adapter_type)) { 212 struct cpl_act_open_req *req = 213 (struct cpl_act_open_req *)skb->head; 214 215 INIT_TP_WR(req, 0); 216 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 217 qid_atid)); 218 req->local_port = csk->saddr.sin_port; 219 req->peer_port = csk->daddr.sin_port; 220 req->local_ip = csk->saddr.sin_addr.s_addr; 221 req->peer_ip = csk->daddr.sin_addr.s_addr; 222 req->opt0 = cpu_to_be64(opt0); 223 req->params = cpu_to_be32(cxgb4_select_ntuple( 224 csk->cdev->ports[csk->port_id], 225 csk->l2t)); 226 opt2 |= RX_FC_VALID_F; 227 req->opt2 = cpu_to_be32(opt2); 228 229 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 230 "csk t4 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 231 csk, &req->local_ip, ntohs(req->local_port), 232 &req->peer_ip, ntohs(req->peer_port), 233 csk->atid, csk->rss_qid); 234 } else if (is_t5(lldi->adapter_type)) { 235 struct cpl_t5_act_open_req *req = 236 (struct cpl_t5_act_open_req *)skb->head; 237 u32 isn = (prandom_u32() & ~7UL) - 1; 238 239 INIT_TP_WR(req, 0); 240 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 241 qid_atid)); 242 req->local_port = csk->saddr.sin_port; 243 req->peer_port = csk->daddr.sin_port; 244 req->local_ip = csk->saddr.sin_addr.s_addr; 245 req->peer_ip = csk->daddr.sin_addr.s_addr; 246 req->opt0 = cpu_to_be64(opt0); 247 req->params = cpu_to_be64(FILTER_TUPLE_V( 248 cxgb4_select_ntuple( 249 csk->cdev->ports[csk->port_id], 250 csk->l2t))); 251 req->rsvd = cpu_to_be32(isn); 252 opt2 |= T5_ISS_VALID; 253 opt2 |= T5_OPT_2_VALID_F; 254 255 req->opt2 = cpu_to_be32(opt2); 256 257 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 258 "csk t5 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 259 csk, &req->local_ip, ntohs(req->local_port), 260 &req->peer_ip, ntohs(req->peer_port), 261 csk->atid, csk->rss_qid); 262 } else { 263 struct cpl_t6_act_open_req *req = 264 (struct cpl_t6_act_open_req *)skb->head; 265 u32 isn = (prandom_u32() & ~7UL) - 1; 266 267 INIT_TP_WR(req, 0); 268 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, 269 qid_atid)); 270 req->local_port = csk->saddr.sin_port; 271 req->peer_port = csk->daddr.sin_port; 272 req->local_ip = csk->saddr.sin_addr.s_addr; 273 req->peer_ip = csk->daddr.sin_addr.s_addr; 274 req->opt0 = cpu_to_be64(opt0); 275 req->params = cpu_to_be64(FILTER_TUPLE_V( 276 cxgb4_select_ntuple( 277 csk->cdev->ports[csk->port_id], 278 csk->l2t))); 279 req->rsvd = cpu_to_be32(isn); 280 281 opt2 |= T5_ISS_VALID; 282 opt2 |= RX_FC_DISABLE_F; 283 opt2 |= T5_OPT_2_VALID_F; 284 285 req->opt2 = cpu_to_be32(opt2); 286 req->rsvd2 = cpu_to_be32(0); 287 req->opt3 = cpu_to_be32(0); 288 289 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 290 "csk t6 0x%p, %pI4:%u-%pI4:%u, atid %d, qid %u.\n", 291 csk, &req->local_ip, ntohs(req->local_port), 292 &req->peer_ip, ntohs(req->peer_port), 293 csk->atid, csk->rss_qid); 294 } 295 296 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 297 298 pr_info_ipaddr("t%d csk 0x%p,%u,0x%lx,%u, rss_qid %u.\n", 299 (&csk->saddr), (&csk->daddr), 300 CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, 301 csk->state, csk->flags, csk->atid, csk->rss_qid); 302 303 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 304 } 305 306 #if IS_ENABLED(CONFIG_IPV6) 307 static void send_act_open_req6(struct cxgbi_sock *csk, struct sk_buff *skb, 308 struct l2t_entry *e) 309 { 310 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 311 int wscale = cxgbi_sock_compute_wscale(csk->mss_idx); 312 unsigned long long opt0; 313 unsigned int opt2; 314 unsigned int qid_atid = ((unsigned int)csk->atid) | 315 (((unsigned int)csk->rss_qid) << 14); 316 317 opt0 = KEEP_ALIVE_F | 318 WND_SCALE_V(wscale) | 319 MSS_IDX_V(csk->mss_idx) | 320 L2T_IDX_V(((struct l2t_entry *)csk->l2t)->idx) | 321 TX_CHAN_V(csk->tx_chan) | 322 SMAC_SEL_V(csk->smac_idx) | 323 ULP_MODE_V(ULP_MODE_ISCSI) | 324 RCV_BUFSIZ_V(csk->rcv_win >> 10); 325 326 opt2 = RX_CHANNEL_V(0) | 327 RSS_QUEUE_VALID_F | 328 RSS_QUEUE_V(csk->rss_qid); 329 330 if (is_t4(lldi->adapter_type)) { 331 struct cpl_act_open_req6 *req = 332 (struct cpl_act_open_req6 *)skb->head; 333 334 INIT_TP_WR(req, 0); 335 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 336 qid_atid)); 337 req->local_port = csk->saddr6.sin6_port; 338 req->peer_port = csk->daddr6.sin6_port; 339 340 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 341 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 342 8); 343 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 344 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 345 8); 346 347 req->opt0 = cpu_to_be64(opt0); 348 349 opt2 |= RX_FC_VALID_F; 350 req->opt2 = cpu_to_be32(opt2); 351 352 req->params = cpu_to_be32(cxgb4_select_ntuple( 353 csk->cdev->ports[csk->port_id], 354 csk->l2t)); 355 } else if (is_t5(lldi->adapter_type)) { 356 struct cpl_t5_act_open_req6 *req = 357 (struct cpl_t5_act_open_req6 *)skb->head; 358 359 INIT_TP_WR(req, 0); 360 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 361 qid_atid)); 362 req->local_port = csk->saddr6.sin6_port; 363 req->peer_port = csk->daddr6.sin6_port; 364 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 365 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 366 8); 367 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 368 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 369 8); 370 req->opt0 = cpu_to_be64(opt0); 371 372 opt2 |= T5_OPT_2_VALID_F; 373 req->opt2 = cpu_to_be32(opt2); 374 375 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( 376 csk->cdev->ports[csk->port_id], 377 csk->l2t))); 378 } else { 379 struct cpl_t6_act_open_req6 *req = 380 (struct cpl_t6_act_open_req6 *)skb->head; 381 382 INIT_TP_WR(req, 0); 383 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, 384 qid_atid)); 385 req->local_port = csk->saddr6.sin6_port; 386 req->peer_port = csk->daddr6.sin6_port; 387 req->local_ip_hi = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr); 388 req->local_ip_lo = *(__be64 *)(csk->saddr6.sin6_addr.s6_addr + 389 8); 390 req->peer_ip_hi = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr); 391 req->peer_ip_lo = *(__be64 *)(csk->daddr6.sin6_addr.s6_addr + 392 8); 393 req->opt0 = cpu_to_be64(opt0); 394 395 opt2 |= RX_FC_DISABLE_F; 396 opt2 |= T5_OPT_2_VALID_F; 397 398 req->opt2 = cpu_to_be32(opt2); 399 400 req->params = cpu_to_be64(FILTER_TUPLE_V(cxgb4_select_ntuple( 401 csk->cdev->ports[csk->port_id], 402 csk->l2t))); 403 404 req->rsvd2 = cpu_to_be32(0); 405 req->opt3 = cpu_to_be32(0); 406 } 407 408 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id); 409 410 pr_info("t%d csk 0x%p,%u,0x%lx,%u, [%pI6]:%u-[%pI6]:%u, rss_qid %u.\n", 411 CHELSIO_CHIP_VERSION(lldi->adapter_type), csk, csk->state, 412 csk->flags, csk->atid, 413 &csk->saddr6.sin6_addr, ntohs(csk->saddr.sin_port), 414 &csk->daddr6.sin6_addr, ntohs(csk->daddr.sin_port), 415 csk->rss_qid); 416 417 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 418 } 419 #endif 420 421 static void send_close_req(struct cxgbi_sock *csk) 422 { 423 struct sk_buff *skb = csk->cpl_close; 424 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 425 unsigned int tid = csk->tid; 426 427 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 428 "csk 0x%p,%u,0x%lx, tid %u.\n", 429 csk, csk->state, csk->flags, csk->tid); 430 csk->cpl_close = NULL; 431 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 432 INIT_TP_WR(req, tid); 433 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 434 req->rsvd = 0; 435 436 cxgbi_sock_skb_entail(csk, skb); 437 if (csk->state >= CTP_ESTABLISHED) 438 push_tx_frames(csk, 1); 439 } 440 441 static void abort_arp_failure(void *handle, struct sk_buff *skb) 442 { 443 struct cxgbi_sock *csk = (struct cxgbi_sock *)handle; 444 struct cpl_abort_req *req; 445 446 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 447 "csk 0x%p,%u,0x%lx, tid %u, abort.\n", 448 csk, csk->state, csk->flags, csk->tid); 449 req = (struct cpl_abort_req *)skb->data; 450 req->cmd = CPL_ABORT_NO_RST; 451 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 452 } 453 454 static void send_abort_req(struct cxgbi_sock *csk) 455 { 456 struct cpl_abort_req *req; 457 struct sk_buff *skb = csk->cpl_abort_req; 458 459 if (unlikely(csk->state == CTP_ABORTING) || !skb || !csk->cdev) 460 return; 461 462 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 463 send_tx_flowc_wr(csk); 464 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 465 } 466 467 cxgbi_sock_set_state(csk, CTP_ABORTING); 468 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 469 cxgbi_sock_purge_write_queue(csk); 470 471 csk->cpl_abort_req = NULL; 472 req = (struct cpl_abort_req *)skb->head; 473 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 474 req->cmd = CPL_ABORT_SEND_RST; 475 t4_set_arp_err_handler(skb, csk, abort_arp_failure); 476 INIT_TP_WR(req, csk->tid); 477 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 478 req->rsvd0 = htonl(csk->snd_nxt); 479 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 480 481 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 482 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 483 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 484 req->rsvd1); 485 486 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 487 } 488 489 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 490 { 491 struct sk_buff *skb = csk->cpl_abort_rpl; 492 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 493 494 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 495 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 496 csk, csk->state, csk->flags, csk->tid, rst_status); 497 498 csk->cpl_abort_rpl = NULL; 499 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 500 INIT_TP_WR(rpl, csk->tid); 501 OPCODE_TID(rpl) = cpu_to_be32(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 502 rpl->cmd = rst_status; 503 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 504 } 505 506 /* 507 * CPL connection rx data ack: host -> 508 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 509 * credits sent. 510 */ 511 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 512 { 513 struct sk_buff *skb; 514 struct cpl_rx_data_ack *req; 515 516 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 517 "csk 0x%p,%u,0x%lx,%u, credit %u.\n", 518 csk, csk->state, csk->flags, csk->tid, credits); 519 520 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 521 if (!skb) { 522 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 523 return 0; 524 } 525 req = (struct cpl_rx_data_ack *)skb->head; 526 527 set_wr_txq(skb, CPL_PRIORITY_ACK, csk->port_id); 528 INIT_TP_WR(req, csk->tid); 529 OPCODE_TID(req) = cpu_to_be32(MK_OPCODE_TID(CPL_RX_DATA_ACK, 530 csk->tid)); 531 req->credit_dack = cpu_to_be32(RX_CREDITS_V(credits) 532 | RX_FORCE_ACK_F); 533 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 534 return credits; 535 } 536 537 /* 538 * sgl_len - calculates the size of an SGL of the given capacity 539 * @n: the number of SGL entries 540 * Calculates the number of flits needed for a scatter/gather list that 541 * can hold the given number of entries. 542 */ 543 static inline unsigned int sgl_len(unsigned int n) 544 { 545 n--; 546 return (3 * n) / 2 + (n & 1) + 2; 547 } 548 549 /* 550 * calc_tx_flits_ofld - calculate # of flits for an offload packet 551 * @skb: the packet 552 * 553 * Returns the number of flits needed for the given offload packet. 554 * These packets are already fully constructed and no additional headers 555 * will be added. 556 */ 557 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb) 558 { 559 unsigned int flits, cnt; 560 561 if (is_ofld_imm(skb)) 562 return DIV_ROUND_UP(skb->len, 8); 563 flits = skb_transport_offset(skb) / 8; 564 cnt = skb_shinfo(skb)->nr_frags; 565 if (skb_tail_pointer(skb) != skb_transport_header(skb)) 566 cnt++; 567 return flits + sgl_len(cnt); 568 } 569 570 #define FLOWC_WR_NPARAMS_MIN 9 571 static inline int tx_flowc_wr_credits(int *nparamsp, int *flowclenp) 572 { 573 int nparams, flowclen16, flowclen; 574 575 nparams = FLOWC_WR_NPARAMS_MIN; 576 flowclen = offsetof(struct fw_flowc_wr, mnemval[nparams]); 577 flowclen16 = DIV_ROUND_UP(flowclen, 16); 578 flowclen = flowclen16 * 16; 579 /* 580 * Return the number of 16-byte credits used by the FlowC request. 581 * Pass back the nparams and actual FlowC length if requested. 582 */ 583 if (nparamsp) 584 *nparamsp = nparams; 585 if (flowclenp) 586 *flowclenp = flowclen; 587 588 return flowclen16; 589 } 590 591 static inline int send_tx_flowc_wr(struct cxgbi_sock *csk) 592 { 593 struct sk_buff *skb; 594 struct fw_flowc_wr *flowc; 595 int nparams, flowclen16, flowclen; 596 597 flowclen16 = tx_flowc_wr_credits(&nparams, &flowclen); 598 skb = alloc_wr(flowclen, 0, GFP_ATOMIC); 599 flowc = (struct fw_flowc_wr *)skb->head; 600 flowc->op_to_nparams = 601 htonl(FW_WR_OP_V(FW_FLOWC_WR) | FW_FLOWC_WR_NPARAMS_V(nparams)); 602 flowc->flowid_len16 = 603 htonl(FW_WR_LEN16_V(flowclen16) | FW_WR_FLOWID_V(csk->tid)); 604 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN; 605 flowc->mnemval[0].val = htonl(csk->cdev->pfvf); 606 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH; 607 flowc->mnemval[1].val = htonl(csk->tx_chan); 608 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT; 609 flowc->mnemval[2].val = htonl(csk->tx_chan); 610 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID; 611 flowc->mnemval[3].val = htonl(csk->rss_qid); 612 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT; 613 flowc->mnemval[4].val = htonl(csk->snd_nxt); 614 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT; 615 flowc->mnemval[5].val = htonl(csk->rcv_nxt); 616 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF; 617 flowc->mnemval[6].val = htonl(csk->snd_win); 618 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS; 619 flowc->mnemval[7].val = htonl(csk->advmss); 620 flowc->mnemval[8].mnemonic = 0; 621 flowc->mnemval[8].val = 0; 622 flowc->mnemval[8].mnemonic = FW_FLOWC_MNEM_TXDATAPLEN_MAX; 623 flowc->mnemval[8].val = 16384; 624 625 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 626 627 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 628 "csk 0x%p, tid 0x%x, %u,%u,%u,%u,%u,%u,%u.\n", 629 csk, csk->tid, 0, csk->tx_chan, csk->rss_qid, 630 csk->snd_nxt, csk->rcv_nxt, csk->snd_win, 631 csk->advmss); 632 633 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 634 635 return flowclen16; 636 } 637 638 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 639 int dlen, int len, u32 credits, int compl) 640 { 641 struct fw_ofld_tx_data_wr *req; 642 unsigned int submode = cxgbi_skcb_ulp_mode(skb) & 3; 643 unsigned int wr_ulp_mode = 0, val; 644 bool imm = is_ofld_imm(skb); 645 646 req = (struct fw_ofld_tx_data_wr *)__skb_push(skb, sizeof(*req)); 647 648 if (imm) { 649 req->op_to_immdlen = htonl(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) | 650 FW_WR_COMPL_F | 651 FW_WR_IMMDLEN_V(dlen)); 652 req->flowid_len16 = htonl(FW_WR_FLOWID_V(csk->tid) | 653 FW_WR_LEN16_V(credits)); 654 } else { 655 req->op_to_immdlen = 656 cpu_to_be32(FW_WR_OP_V(FW_OFLD_TX_DATA_WR) | 657 FW_WR_COMPL_F | 658 FW_WR_IMMDLEN_V(0)); 659 req->flowid_len16 = 660 cpu_to_be32(FW_WR_FLOWID_V(csk->tid) | 661 FW_WR_LEN16_V(credits)); 662 } 663 if (submode) 664 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP2_MODE_ISCSI) | 665 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode); 666 val = skb_peek(&csk->write_queue) ? 0 : 1; 667 req->tunnel_to_proxy = htonl(wr_ulp_mode | 668 FW_OFLD_TX_DATA_WR_SHOVE_V(val)); 669 req->plen = htonl(len); 670 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) 671 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 672 } 673 674 static void arp_failure_skb_discard(void *handle, struct sk_buff *skb) 675 { 676 kfree_skb(skb); 677 } 678 679 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 680 { 681 int total_size = 0; 682 struct sk_buff *skb; 683 684 if (unlikely(csk->state < CTP_ESTABLISHED || 685 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 686 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK | 687 1 << CXGBI_DBG_PDU_TX, 688 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 689 csk, csk->state, csk->flags, csk->tid); 690 return 0; 691 } 692 693 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { 694 int dlen = skb->len; 695 int len = skb->len; 696 unsigned int credits_needed; 697 int flowclen16 = 0; 698 699 skb_reset_transport_header(skb); 700 if (is_ofld_imm(skb)) 701 credits_needed = DIV_ROUND_UP(dlen, 16); 702 else 703 credits_needed = DIV_ROUND_UP( 704 8 * calc_tx_flits_ofld(skb), 705 16); 706 707 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) 708 credits_needed += DIV_ROUND_UP( 709 sizeof(struct fw_ofld_tx_data_wr), 710 16); 711 712 /* 713 * Assumes the initial credits is large enough to support 714 * fw_flowc_wr plus largest possible first payload 715 */ 716 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 717 flowclen16 = send_tx_flowc_wr(csk); 718 csk->wr_cred -= flowclen16; 719 csk->wr_una_cred += flowclen16; 720 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 721 } 722 723 if (csk->wr_cred < credits_needed) { 724 log_debug(1 << CXGBI_DBG_PDU_TX, 725 "csk 0x%p, skb %u/%u, wr %d < %u.\n", 726 csk, skb->len, skb->data_len, 727 credits_needed, csk->wr_cred); 728 break; 729 } 730 __skb_unlink(skb, &csk->write_queue); 731 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 732 skb->csum = credits_needed + flowclen16; 733 csk->wr_cred -= credits_needed; 734 csk->wr_una_cred += credits_needed; 735 cxgbi_sock_enqueue_wr(csk, skb); 736 737 log_debug(1 << CXGBI_DBG_PDU_TX, 738 "csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n", 739 csk, skb->len, skb->data_len, credits_needed, 740 csk->wr_cred, csk->wr_una_cred); 741 742 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 743 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 744 make_tx_data_wr(csk, skb, dlen, len, credits_needed, 745 req_completion); 746 csk->snd_nxt += len; 747 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 748 } else if (cxgbi_skcb_test_flag(skb, SKCBF_TX_FLAG_COMPL) && 749 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) { 750 struct cpl_close_con_req *req = 751 (struct cpl_close_con_req *)skb->data; 752 req->wr.wr_hi |= htonl(FW_WR_COMPL_F); 753 } 754 total_size += skb->truesize; 755 t4_set_arp_err_handler(skb, csk, arp_failure_skb_discard); 756 757 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 758 "csk 0x%p,%u,0x%lx,%u, skb 0x%p, %u.\n", 759 csk, csk->state, csk->flags, csk->tid, skb, len); 760 761 cxgb4_l2t_send(csk->cdev->ports[csk->port_id], skb, csk->l2t); 762 } 763 return total_size; 764 } 765 766 static inline void free_atid(struct cxgbi_sock *csk) 767 { 768 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 769 770 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 771 cxgb4_free_atid(lldi->tids, csk->atid); 772 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 773 cxgbi_sock_put(csk); 774 } 775 } 776 777 static void do_act_establish(struct cxgbi_device *cdev, struct sk_buff *skb) 778 { 779 struct cxgbi_sock *csk; 780 struct cpl_act_establish *req = (struct cpl_act_establish *)skb->data; 781 unsigned short tcp_opt = ntohs(req->tcp_opt); 782 unsigned int tid = GET_TID(req); 783 unsigned int atid = TID_TID_G(ntohl(req->tos_atid)); 784 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 785 struct tid_info *t = lldi->tids; 786 u32 rcv_isn = be32_to_cpu(req->rcv_isn); 787 788 csk = lookup_atid(t, atid); 789 if (unlikely(!csk)) { 790 pr_err("NO conn. for atid %u, cdev 0x%p.\n", atid, cdev); 791 goto rel_skb; 792 } 793 794 if (csk->atid != atid) { 795 pr_err("bad conn atid %u, csk 0x%p,%u,0x%lx,tid %u, atid %u.\n", 796 atid, csk, csk->state, csk->flags, csk->tid, csk->atid); 797 goto rel_skb; 798 } 799 800 pr_info_ipaddr("atid 0x%x, tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 801 (&csk->saddr), (&csk->daddr), 802 atid, tid, csk, csk->state, csk->flags, rcv_isn); 803 804 module_put(THIS_MODULE); 805 806 cxgbi_sock_get(csk); 807 csk->tid = tid; 808 cxgb4_insert_tid(lldi->tids, csk, tid); 809 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 810 811 free_atid(csk); 812 813 spin_lock_bh(&csk->lock); 814 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 815 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 816 csk, csk->state, csk->flags, csk->tid); 817 818 if (csk->retry_timer.function) { 819 del_timer(&csk->retry_timer); 820 csk->retry_timer.function = NULL; 821 } 822 823 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 824 /* 825 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't 826 * pass through opt0. 827 */ 828 if (csk->rcv_win > (RCV_BUFSIZ_MASK << 10)) 829 csk->rcv_wup -= csk->rcv_win - (RCV_BUFSIZ_MASK << 10); 830 831 csk->advmss = lldi->mtus[TCPOPT_MSS_G(tcp_opt)] - 40; 832 if (TCPOPT_TSTAMP_G(tcp_opt)) 833 csk->advmss -= 12; 834 if (csk->advmss < 128) 835 csk->advmss = 128; 836 837 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 838 "csk 0x%p, mss_idx %u, advmss %u.\n", 839 csk, TCPOPT_MSS_G(tcp_opt), csk->advmss); 840 841 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 842 843 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 844 send_abort_req(csk); 845 else { 846 if (skb_queue_len(&csk->write_queue)) 847 push_tx_frames(csk, 0); 848 cxgbi_conn_tx_open(csk); 849 } 850 spin_unlock_bh(&csk->lock); 851 852 rel_skb: 853 __kfree_skb(skb); 854 } 855 856 static int act_open_rpl_status_to_errno(int status) 857 { 858 switch (status) { 859 case CPL_ERR_CONN_RESET: 860 return -ECONNREFUSED; 861 case CPL_ERR_ARP_MISS: 862 return -EHOSTUNREACH; 863 case CPL_ERR_CONN_TIMEDOUT: 864 return -ETIMEDOUT; 865 case CPL_ERR_TCAM_FULL: 866 return -ENOMEM; 867 case CPL_ERR_CONN_EXIST: 868 return -EADDRINUSE; 869 default: 870 return -EIO; 871 } 872 } 873 874 static void csk_act_open_retry_timer(unsigned long data) 875 { 876 struct sk_buff *skb = NULL; 877 struct cxgbi_sock *csk = (struct cxgbi_sock *)data; 878 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(csk->cdev); 879 void (*send_act_open_func)(struct cxgbi_sock *, struct sk_buff *, 880 struct l2t_entry *); 881 int t4 = is_t4(lldi->adapter_type), size, size6; 882 883 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 884 "csk 0x%p,%u,0x%lx,%u.\n", 885 csk, csk->state, csk->flags, csk->tid); 886 887 cxgbi_sock_get(csk); 888 spin_lock_bh(&csk->lock); 889 890 if (t4) { 891 size = sizeof(struct cpl_act_open_req); 892 size6 = sizeof(struct cpl_act_open_req6); 893 } else { 894 size = sizeof(struct cpl_t5_act_open_req); 895 size6 = sizeof(struct cpl_t5_act_open_req6); 896 } 897 898 if (csk->csk_family == AF_INET) { 899 send_act_open_func = send_act_open_req; 900 skb = alloc_wr(size, 0, GFP_ATOMIC); 901 #if IS_ENABLED(CONFIG_IPV6) 902 } else { 903 send_act_open_func = send_act_open_req6; 904 skb = alloc_wr(size6, 0, GFP_ATOMIC); 905 #endif 906 } 907 908 if (!skb) 909 cxgbi_sock_fail_act_open(csk, -ENOMEM); 910 else { 911 skb->sk = (struct sock *)csk; 912 t4_set_arp_err_handler(skb, csk, 913 cxgbi_sock_act_open_req_arp_failure); 914 send_act_open_func(csk, skb, csk->l2t); 915 } 916 917 spin_unlock_bh(&csk->lock); 918 cxgbi_sock_put(csk); 919 920 } 921 922 static inline bool is_neg_adv(unsigned int status) 923 { 924 return status == CPL_ERR_RTX_NEG_ADVICE || 925 status == CPL_ERR_KEEPALV_NEG_ADVICE || 926 status == CPL_ERR_PERSIST_NEG_ADVICE; 927 } 928 929 static void do_act_open_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 930 { 931 struct cxgbi_sock *csk; 932 struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)skb->data; 933 unsigned int tid = GET_TID(rpl); 934 unsigned int atid = 935 TID_TID_G(AOPEN_ATID_G(be32_to_cpu(rpl->atid_status))); 936 unsigned int status = AOPEN_STATUS_G(be32_to_cpu(rpl->atid_status)); 937 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 938 struct tid_info *t = lldi->tids; 939 940 csk = lookup_atid(t, atid); 941 if (unlikely(!csk)) { 942 pr_err("NO matching conn. atid %u, tid %u.\n", atid, tid); 943 goto rel_skb; 944 } 945 946 pr_info_ipaddr("tid %u/%u, status %u.\n" 947 "csk 0x%p,%u,0x%lx. ", (&csk->saddr), (&csk->daddr), 948 atid, tid, status, csk, csk->state, csk->flags); 949 950 if (is_neg_adv(status)) 951 goto rel_skb; 952 953 module_put(THIS_MODULE); 954 955 if (status && status != CPL_ERR_TCAM_FULL && 956 status != CPL_ERR_CONN_EXIST && 957 status != CPL_ERR_ARP_MISS) 958 cxgb4_remove_tid(lldi->tids, csk->port_id, GET_TID(rpl)); 959 960 cxgbi_sock_get(csk); 961 spin_lock_bh(&csk->lock); 962 963 if (status == CPL_ERR_CONN_EXIST && 964 csk->retry_timer.function != csk_act_open_retry_timer) { 965 csk->retry_timer.function = csk_act_open_retry_timer; 966 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 967 } else 968 cxgbi_sock_fail_act_open(csk, 969 act_open_rpl_status_to_errno(status)); 970 971 spin_unlock_bh(&csk->lock); 972 cxgbi_sock_put(csk); 973 rel_skb: 974 __kfree_skb(skb); 975 } 976 977 static void do_peer_close(struct cxgbi_device *cdev, struct sk_buff *skb) 978 { 979 struct cxgbi_sock *csk; 980 struct cpl_peer_close *req = (struct cpl_peer_close *)skb->data; 981 unsigned int tid = GET_TID(req); 982 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 983 struct tid_info *t = lldi->tids; 984 985 csk = lookup_tid(t, tid); 986 if (unlikely(!csk)) { 987 pr_err("can't find connection for tid %u.\n", tid); 988 goto rel_skb; 989 } 990 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 991 (&csk->saddr), (&csk->daddr), 992 csk, csk->state, csk->flags, csk->tid); 993 cxgbi_sock_rcv_peer_close(csk); 994 rel_skb: 995 __kfree_skb(skb); 996 } 997 998 static void do_close_con_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 999 { 1000 struct cxgbi_sock *csk; 1001 struct cpl_close_con_rpl *rpl = (struct cpl_close_con_rpl *)skb->data; 1002 unsigned int tid = GET_TID(rpl); 1003 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1004 struct tid_info *t = lldi->tids; 1005 1006 csk = lookup_tid(t, tid); 1007 if (unlikely(!csk)) { 1008 pr_err("can't find connection for tid %u.\n", tid); 1009 goto rel_skb; 1010 } 1011 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u.\n", 1012 (&csk->saddr), (&csk->daddr), 1013 csk, csk->state, csk->flags, csk->tid); 1014 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 1015 rel_skb: 1016 __kfree_skb(skb); 1017 } 1018 1019 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 1020 int *need_rst) 1021 { 1022 switch (abort_reason) { 1023 case CPL_ERR_BAD_SYN: /* fall through */ 1024 case CPL_ERR_CONN_RESET: 1025 return csk->state > CTP_ESTABLISHED ? 1026 -EPIPE : -ECONNRESET; 1027 case CPL_ERR_XMIT_TIMEDOUT: 1028 case CPL_ERR_PERSIST_TIMEDOUT: 1029 case CPL_ERR_FINWAIT2_TIMEDOUT: 1030 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1031 return -ETIMEDOUT; 1032 default: 1033 return -EIO; 1034 } 1035 } 1036 1037 static void do_abort_req_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 1038 { 1039 struct cxgbi_sock *csk; 1040 struct cpl_abort_req_rss *req = (struct cpl_abort_req_rss *)skb->data; 1041 unsigned int tid = GET_TID(req); 1042 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1043 struct tid_info *t = lldi->tids; 1044 int rst_status = CPL_ABORT_NO_RST; 1045 1046 csk = lookup_tid(t, tid); 1047 if (unlikely(!csk)) { 1048 pr_err("can't find connection for tid %u.\n", tid); 1049 goto rel_skb; 1050 } 1051 1052 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 1053 (&csk->saddr), (&csk->daddr), 1054 csk, csk->state, csk->flags, csk->tid, req->status); 1055 1056 if (is_neg_adv(req->status)) 1057 goto rel_skb; 1058 1059 cxgbi_sock_get(csk); 1060 spin_lock_bh(&csk->lock); 1061 1062 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 1063 1064 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 1065 send_tx_flowc_wr(csk); 1066 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 1067 } 1068 1069 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 1070 cxgbi_sock_set_state(csk, CTP_ABORTING); 1071 1072 send_abort_rpl(csk, rst_status); 1073 1074 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 1075 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 1076 cxgbi_sock_closed(csk); 1077 } 1078 1079 spin_unlock_bh(&csk->lock); 1080 cxgbi_sock_put(csk); 1081 rel_skb: 1082 __kfree_skb(skb); 1083 } 1084 1085 static void do_abort_rpl_rss(struct cxgbi_device *cdev, struct sk_buff *skb) 1086 { 1087 struct cxgbi_sock *csk; 1088 struct cpl_abort_rpl_rss *rpl = (struct cpl_abort_rpl_rss *)skb->data; 1089 unsigned int tid = GET_TID(rpl); 1090 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1091 struct tid_info *t = lldi->tids; 1092 1093 csk = lookup_tid(t, tid); 1094 if (!csk) 1095 goto rel_skb; 1096 1097 if (csk) 1098 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u, status %u.\n", 1099 (&csk->saddr), (&csk->daddr), csk, 1100 csk->state, csk->flags, csk->tid, rpl->status); 1101 1102 if (rpl->status == CPL_ERR_ABORT_FAILED) 1103 goto rel_skb; 1104 1105 cxgbi_sock_rcv_abort_rpl(csk); 1106 rel_skb: 1107 __kfree_skb(skb); 1108 } 1109 1110 static void do_rx_data(struct cxgbi_device *cdev, struct sk_buff *skb) 1111 { 1112 struct cxgbi_sock *csk; 1113 struct cpl_rx_data *cpl = (struct cpl_rx_data *)skb->data; 1114 unsigned int tid = GET_TID(cpl); 1115 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1116 struct tid_info *t = lldi->tids; 1117 1118 csk = lookup_tid(t, tid); 1119 if (!csk) { 1120 pr_err("can't find connection for tid %u.\n", tid); 1121 } else { 1122 /* not expecting this, reset the connection. */ 1123 pr_err("csk 0x%p, tid %u, rcv cpl_rx_data.\n", csk, tid); 1124 spin_lock_bh(&csk->lock); 1125 send_abort_req(csk); 1126 spin_unlock_bh(&csk->lock); 1127 } 1128 __kfree_skb(skb); 1129 } 1130 1131 static void do_rx_iscsi_hdr(struct cxgbi_device *cdev, struct sk_buff *skb) 1132 { 1133 struct cxgbi_sock *csk; 1134 struct cpl_iscsi_hdr *cpl = (struct cpl_iscsi_hdr *)skb->data; 1135 unsigned short pdu_len_ddp = be16_to_cpu(cpl->pdu_len_ddp); 1136 unsigned int tid = GET_TID(cpl); 1137 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1138 struct tid_info *t = lldi->tids; 1139 1140 csk = lookup_tid(t, tid); 1141 if (unlikely(!csk)) { 1142 pr_err("can't find conn. for tid %u.\n", tid); 1143 goto rel_skb; 1144 } 1145 1146 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1147 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p,%u, 0x%x.\n", 1148 csk, csk->state, csk->flags, csk->tid, skb, skb->len, 1149 pdu_len_ddp); 1150 1151 spin_lock_bh(&csk->lock); 1152 1153 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1154 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1155 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1156 csk, csk->state, csk->flags, csk->tid); 1157 if (csk->state != CTP_ABORTING) 1158 goto abort_conn; 1159 else 1160 goto discard; 1161 } 1162 1163 cxgbi_skcb_tcp_seq(skb) = ntohl(cpl->seq); 1164 cxgbi_skcb_flags(skb) = 0; 1165 1166 skb_reset_transport_header(skb); 1167 __skb_pull(skb, sizeof(*cpl)); 1168 __pskb_trim(skb, ntohs(cpl->len)); 1169 1170 if (!csk->skb_ulp_lhdr) { 1171 unsigned char *bhs; 1172 unsigned int hlen, dlen, plen; 1173 1174 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1175 "csk 0x%p,%u,0x%lx, tid %u, skb 0x%p header.\n", 1176 csk, csk->state, csk->flags, csk->tid, skb); 1177 csk->skb_ulp_lhdr = skb; 1178 cxgbi_skcb_set_flag(skb, SKCBF_RX_HDR); 1179 1180 if (cxgbi_skcb_tcp_seq(skb) != csk->rcv_nxt) { 1181 pr_info("tid %u, CPL_ISCSI_HDR, bad seq, 0x%x/0x%x.\n", 1182 csk->tid, cxgbi_skcb_tcp_seq(skb), 1183 csk->rcv_nxt); 1184 goto abort_conn; 1185 } 1186 1187 bhs = skb->data; 1188 hlen = ntohs(cpl->len); 1189 dlen = ntohl(*(unsigned int *)(bhs + 4)) & 0xFFFFFF; 1190 1191 plen = ISCSI_PDU_LEN_G(pdu_len_ddp); 1192 if (is_t4(lldi->adapter_type)) 1193 plen -= 40; 1194 1195 if ((hlen + dlen) != plen) { 1196 pr_info("tid 0x%x, CPL_ISCSI_HDR, pdu len " 1197 "mismatch %u != %u + %u, seq 0x%x.\n", 1198 csk->tid, plen, hlen, dlen, 1199 cxgbi_skcb_tcp_seq(skb)); 1200 goto abort_conn; 1201 } 1202 1203 cxgbi_skcb_rx_pdulen(skb) = (hlen + dlen + 3) & (~0x3); 1204 if (dlen) 1205 cxgbi_skcb_rx_pdulen(skb) += csk->dcrc_len; 1206 csk->rcv_nxt += cxgbi_skcb_rx_pdulen(skb); 1207 1208 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1209 "csk 0x%p, skb 0x%p, 0x%x,%u+%u,0x%x,0x%x.\n", 1210 csk, skb, *bhs, hlen, dlen, 1211 ntohl(*((unsigned int *)(bhs + 16))), 1212 ntohl(*((unsigned int *)(bhs + 24)))); 1213 1214 } else { 1215 struct sk_buff *lskb = csk->skb_ulp_lhdr; 1216 1217 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA); 1218 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1219 "csk 0x%p,%u,0x%lx, skb 0x%p data, 0x%p.\n", 1220 csk, csk->state, csk->flags, skb, lskb); 1221 } 1222 1223 __skb_queue_tail(&csk->receive_queue, skb); 1224 spin_unlock_bh(&csk->lock); 1225 return; 1226 1227 abort_conn: 1228 send_abort_req(csk); 1229 discard: 1230 spin_unlock_bh(&csk->lock); 1231 rel_skb: 1232 __kfree_skb(skb); 1233 } 1234 1235 static void do_rx_data_ddp(struct cxgbi_device *cdev, 1236 struct sk_buff *skb) 1237 { 1238 struct cxgbi_sock *csk; 1239 struct sk_buff *lskb; 1240 struct cpl_rx_data_ddp *rpl = (struct cpl_rx_data_ddp *)skb->data; 1241 unsigned int tid = GET_TID(rpl); 1242 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1243 struct tid_info *t = lldi->tids; 1244 unsigned int status = ntohl(rpl->ddpvld); 1245 1246 csk = lookup_tid(t, tid); 1247 if (unlikely(!csk)) { 1248 pr_err("can't find connection for tid %u.\n", tid); 1249 goto rel_skb; 1250 } 1251 1252 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 1253 "csk 0x%p,%u,0x%lx, skb 0x%p,0x%x, lhdr 0x%p.\n", 1254 csk, csk->state, csk->flags, skb, status, csk->skb_ulp_lhdr); 1255 1256 spin_lock_bh(&csk->lock); 1257 1258 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 1259 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1260 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 1261 csk, csk->state, csk->flags, csk->tid); 1262 if (csk->state != CTP_ABORTING) 1263 goto abort_conn; 1264 else 1265 goto discard; 1266 } 1267 1268 if (!csk->skb_ulp_lhdr) { 1269 pr_err("tid 0x%x, rcv RX_DATA_DDP w/o pdu bhs.\n", csk->tid); 1270 goto abort_conn; 1271 } 1272 1273 lskb = csk->skb_ulp_lhdr; 1274 csk->skb_ulp_lhdr = NULL; 1275 1276 cxgbi_skcb_rx_ddigest(lskb) = ntohl(rpl->ulp_crc); 1277 1278 if (ntohs(rpl->len) != cxgbi_skcb_rx_pdulen(lskb)) 1279 pr_info("tid 0x%x, RX_DATA_DDP pdulen %u != %u.\n", 1280 csk->tid, ntohs(rpl->len), cxgbi_skcb_rx_pdulen(lskb)); 1281 1282 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) { 1283 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, hcrc bad 0x%lx.\n", 1284 csk, lskb, status, cxgbi_skcb_flags(lskb)); 1285 cxgbi_skcb_set_flag(lskb, SKCBF_RX_HCRC_ERR); 1286 } 1287 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) { 1288 pr_info("csk 0x%p, lhdr 0x%p, status 0x%x, dcrc bad 0x%lx.\n", 1289 csk, lskb, status, cxgbi_skcb_flags(lskb)); 1290 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DCRC_ERR); 1291 } 1292 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) { 1293 log_debug(1 << CXGBI_DBG_PDU_RX, 1294 "csk 0x%p, lhdr 0x%p, status 0x%x, pad bad.\n", 1295 csk, lskb, status); 1296 cxgbi_skcb_set_flag(lskb, SKCBF_RX_PAD_ERR); 1297 } 1298 if ((status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) && 1299 !cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA)) { 1300 log_debug(1 << CXGBI_DBG_PDU_RX, 1301 "csk 0x%p, lhdr 0x%p, 0x%x, data ddp'ed.\n", 1302 csk, lskb, status); 1303 cxgbi_skcb_set_flag(lskb, SKCBF_RX_DATA_DDPD); 1304 } 1305 log_debug(1 << CXGBI_DBG_PDU_RX, 1306 "csk 0x%p, lskb 0x%p, f 0x%lx.\n", 1307 csk, lskb, cxgbi_skcb_flags(lskb)); 1308 1309 cxgbi_skcb_set_flag(lskb, SKCBF_RX_STATUS); 1310 cxgbi_conn_pdu_ready(csk); 1311 spin_unlock_bh(&csk->lock); 1312 goto rel_skb; 1313 1314 abort_conn: 1315 send_abort_req(csk); 1316 discard: 1317 spin_unlock_bh(&csk->lock); 1318 rel_skb: 1319 __kfree_skb(skb); 1320 } 1321 1322 static void do_fw4_ack(struct cxgbi_device *cdev, struct sk_buff *skb) 1323 { 1324 struct cxgbi_sock *csk; 1325 struct cpl_fw4_ack *rpl = (struct cpl_fw4_ack *)skb->data; 1326 unsigned int tid = GET_TID(rpl); 1327 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1328 struct tid_info *t = lldi->tids; 1329 1330 csk = lookup_tid(t, tid); 1331 if (unlikely(!csk)) 1332 pr_err("can't find connection for tid %u.\n", tid); 1333 else { 1334 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1335 "csk 0x%p,%u,0x%lx,%u.\n", 1336 csk, csk->state, csk->flags, csk->tid); 1337 cxgbi_sock_rcv_wr_ack(csk, rpl->credits, ntohl(rpl->snd_una), 1338 rpl->seq_vld); 1339 } 1340 __kfree_skb(skb); 1341 } 1342 1343 static void do_set_tcb_rpl(struct cxgbi_device *cdev, struct sk_buff *skb) 1344 { 1345 struct cpl_set_tcb_rpl *rpl = (struct cpl_set_tcb_rpl *)skb->data; 1346 unsigned int tid = GET_TID(rpl); 1347 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1348 struct tid_info *t = lldi->tids; 1349 struct cxgbi_sock *csk; 1350 1351 csk = lookup_tid(t, tid); 1352 if (!csk) 1353 pr_err("can't find conn. for tid %u.\n", tid); 1354 1355 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1356 "csk 0x%p,%u,%lx,%u, status 0x%x.\n", 1357 csk, csk->state, csk->flags, csk->tid, rpl->status); 1358 1359 if (rpl->status != CPL_ERR_NONE) 1360 pr_err("csk 0x%p,%u, SET_TCB_RPL status %u.\n", 1361 csk, tid, rpl->status); 1362 1363 __kfree_skb(skb); 1364 } 1365 1366 static int alloc_cpls(struct cxgbi_sock *csk) 1367 { 1368 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 1369 0, GFP_KERNEL); 1370 if (!csk->cpl_close) 1371 return -ENOMEM; 1372 1373 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 1374 0, GFP_KERNEL); 1375 if (!csk->cpl_abort_req) 1376 goto free_cpls; 1377 1378 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 1379 0, GFP_KERNEL); 1380 if (!csk->cpl_abort_rpl) 1381 goto free_cpls; 1382 return 0; 1383 1384 free_cpls: 1385 cxgbi_sock_free_cpl_skbs(csk); 1386 return -ENOMEM; 1387 } 1388 1389 static inline void l2t_put(struct cxgbi_sock *csk) 1390 { 1391 if (csk->l2t) { 1392 cxgb4_l2t_release(csk->l2t); 1393 csk->l2t = NULL; 1394 cxgbi_sock_put(csk); 1395 } 1396 } 1397 1398 static void release_offload_resources(struct cxgbi_sock *csk) 1399 { 1400 struct cxgb4_lld_info *lldi; 1401 #if IS_ENABLED(CONFIG_IPV6) 1402 struct net_device *ndev = csk->cdev->ports[csk->port_id]; 1403 #endif 1404 1405 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1406 "csk 0x%p,%u,0x%lx,%u.\n", 1407 csk, csk->state, csk->flags, csk->tid); 1408 1409 cxgbi_sock_free_cpl_skbs(csk); 1410 if (csk->wr_cred != csk->wr_max_cred) { 1411 cxgbi_sock_purge_wr_queue(csk); 1412 cxgbi_sock_reset_wr_list(csk); 1413 } 1414 1415 l2t_put(csk); 1416 #if IS_ENABLED(CONFIG_IPV6) 1417 if (csk->csk_family == AF_INET6) 1418 cxgb4_clip_release(ndev, 1419 (const u32 *)&csk->saddr6.sin6_addr, 1); 1420 #endif 1421 1422 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 1423 free_atid(csk); 1424 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 1425 lldi = cxgbi_cdev_priv(csk->cdev); 1426 cxgb4_remove_tid(lldi->tids, 0, csk->tid); 1427 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 1428 cxgbi_sock_put(csk); 1429 } 1430 csk->dst = NULL; 1431 csk->cdev = NULL; 1432 } 1433 1434 static int init_act_open(struct cxgbi_sock *csk) 1435 { 1436 struct cxgbi_device *cdev = csk->cdev; 1437 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1438 struct net_device *ndev = cdev->ports[csk->port_id]; 1439 struct sk_buff *skb = NULL; 1440 struct neighbour *n = NULL; 1441 void *daddr; 1442 unsigned int step; 1443 unsigned int size, size6; 1444 unsigned int linkspeed; 1445 unsigned int rcv_winf, snd_winf; 1446 1447 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1448 "csk 0x%p,%u,0x%lx,%u.\n", 1449 csk, csk->state, csk->flags, csk->tid); 1450 1451 if (csk->csk_family == AF_INET) 1452 daddr = &csk->daddr.sin_addr.s_addr; 1453 #if IS_ENABLED(CONFIG_IPV6) 1454 else if (csk->csk_family == AF_INET6) 1455 daddr = &csk->daddr6.sin6_addr; 1456 #endif 1457 else { 1458 pr_err("address family 0x%x not supported\n", csk->csk_family); 1459 goto rel_resource; 1460 } 1461 1462 n = dst_neigh_lookup(csk->dst, daddr); 1463 1464 if (!n) { 1465 pr_err("%s, can't get neighbour of csk->dst.\n", ndev->name); 1466 goto rel_resource; 1467 } 1468 1469 csk->atid = cxgb4_alloc_atid(lldi->tids, csk); 1470 if (csk->atid < 0) { 1471 pr_err("%s, NO atid available.\n", ndev->name); 1472 goto rel_resource_without_clip; 1473 } 1474 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 1475 cxgbi_sock_get(csk); 1476 1477 csk->l2t = cxgb4_l2t_get(lldi->l2t, n, ndev, 0); 1478 if (!csk->l2t) { 1479 pr_err("%s, cannot alloc l2t.\n", ndev->name); 1480 goto rel_resource_without_clip; 1481 } 1482 cxgbi_sock_get(csk); 1483 1484 #if IS_ENABLED(CONFIG_IPV6) 1485 if (csk->csk_family == AF_INET6) 1486 cxgb4_clip_get(ndev, (const u32 *)&csk->saddr6.sin6_addr, 1); 1487 #endif 1488 1489 if (is_t4(lldi->adapter_type)) { 1490 size = sizeof(struct cpl_act_open_req); 1491 size6 = sizeof(struct cpl_act_open_req6); 1492 } else if (is_t5(lldi->adapter_type)) { 1493 size = sizeof(struct cpl_t5_act_open_req); 1494 size6 = sizeof(struct cpl_t5_act_open_req6); 1495 } else { 1496 size = sizeof(struct cpl_t6_act_open_req); 1497 size6 = sizeof(struct cpl_t6_act_open_req6); 1498 } 1499 1500 if (csk->csk_family == AF_INET) 1501 skb = alloc_wr(size, 0, GFP_NOIO); 1502 #if IS_ENABLED(CONFIG_IPV6) 1503 else 1504 skb = alloc_wr(size6, 0, GFP_NOIO); 1505 #endif 1506 1507 if (!skb) 1508 goto rel_resource; 1509 skb->sk = (struct sock *)csk; 1510 t4_set_arp_err_handler(skb, csk, cxgbi_sock_act_open_req_arp_failure); 1511 1512 if (!csk->mtu) 1513 csk->mtu = dst_mtu(csk->dst); 1514 cxgb4_best_mtu(lldi->mtus, csk->mtu, &csk->mss_idx); 1515 csk->tx_chan = cxgb4_port_chan(ndev); 1516 csk->smac_idx = cxgb4_tp_smt_idx(lldi->adapter_type, 1517 cxgb4_port_viid(ndev)); 1518 step = lldi->ntxq / lldi->nchan; 1519 csk->txq_idx = cxgb4_port_idx(ndev) * step; 1520 step = lldi->nrxq / lldi->nchan; 1521 csk->rss_qid = lldi->rxq_ids[cxgb4_port_idx(ndev) * step]; 1522 linkspeed = ((struct port_info *)netdev_priv(ndev))->link_cfg.speed; 1523 csk->snd_win = cxgb4i_snd_win; 1524 csk->rcv_win = cxgb4i_rcv_win; 1525 if (cxgb4i_rcv_win <= 0) { 1526 csk->rcv_win = CXGB4I_DEFAULT_10G_RCV_WIN; 1527 rcv_winf = linkspeed / SPEED_10000; 1528 if (rcv_winf) 1529 csk->rcv_win *= rcv_winf; 1530 } 1531 if (cxgb4i_snd_win <= 0) { 1532 csk->snd_win = CXGB4I_DEFAULT_10G_SND_WIN; 1533 snd_winf = linkspeed / SPEED_10000; 1534 if (snd_winf) 1535 csk->snd_win *= snd_winf; 1536 } 1537 csk->wr_cred = lldi->wr_cred - 1538 DIV_ROUND_UP(sizeof(struct cpl_abort_req), 16); 1539 csk->wr_max_cred = csk->wr_cred; 1540 csk->wr_una_cred = 0; 1541 cxgbi_sock_reset_wr_list(csk); 1542 csk->err = 0; 1543 1544 pr_info_ipaddr("csk 0x%p,%u,0x%lx,%u,%u,%u, mtu %u,%u, smac %u.\n", 1545 (&csk->saddr), (&csk->daddr), csk, csk->state, 1546 csk->flags, csk->tx_chan, csk->txq_idx, csk->rss_qid, 1547 csk->mtu, csk->mss_idx, csk->smac_idx); 1548 1549 /* must wait for either a act_open_rpl or act_open_establish */ 1550 try_module_get(THIS_MODULE); 1551 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1552 if (csk->csk_family == AF_INET) 1553 send_act_open_req(csk, skb, csk->l2t); 1554 #if IS_ENABLED(CONFIG_IPV6) 1555 else 1556 send_act_open_req6(csk, skb, csk->l2t); 1557 #endif 1558 neigh_release(n); 1559 1560 return 0; 1561 1562 rel_resource: 1563 #if IS_ENABLED(CONFIG_IPV6) 1564 if (csk->csk_family == AF_INET6) 1565 cxgb4_clip_release(ndev, 1566 (const u32 *)&csk->saddr6.sin6_addr, 1); 1567 #endif 1568 rel_resource_without_clip: 1569 if (n) 1570 neigh_release(n); 1571 if (skb) 1572 __kfree_skb(skb); 1573 return -EINVAL; 1574 } 1575 1576 static cxgb4i_cplhandler_func cxgb4i_cplhandlers[NUM_CPL_CMDS] = { 1577 [CPL_ACT_ESTABLISH] = do_act_establish, 1578 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1579 [CPL_PEER_CLOSE] = do_peer_close, 1580 [CPL_ABORT_REQ_RSS] = do_abort_req_rss, 1581 [CPL_ABORT_RPL_RSS] = do_abort_rpl_rss, 1582 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1583 [CPL_FW4_ACK] = do_fw4_ack, 1584 [CPL_ISCSI_HDR] = do_rx_iscsi_hdr, 1585 [CPL_ISCSI_DATA] = do_rx_iscsi_hdr, 1586 [CPL_SET_TCB_RPL] = do_set_tcb_rpl, 1587 [CPL_RX_DATA_DDP] = do_rx_data_ddp, 1588 [CPL_RX_ISCSI_DDP] = do_rx_data_ddp, 1589 [CPL_RX_DATA] = do_rx_data, 1590 }; 1591 1592 static int cxgb4i_ofld_init(struct cxgbi_device *cdev) 1593 { 1594 int rc; 1595 1596 if (cxgb4i_max_connect > CXGB4I_MAX_CONN) 1597 cxgb4i_max_connect = CXGB4I_MAX_CONN; 1598 1599 rc = cxgbi_device_portmap_create(cdev, cxgb4i_sport_base, 1600 cxgb4i_max_connect); 1601 if (rc < 0) 1602 return rc; 1603 1604 cdev->csk_release_offload_resources = release_offload_resources; 1605 cdev->csk_push_tx_frames = push_tx_frames; 1606 cdev->csk_send_abort_req = send_abort_req; 1607 cdev->csk_send_close_req = send_close_req; 1608 cdev->csk_send_rx_credits = send_rx_credits; 1609 cdev->csk_alloc_cpls = alloc_cpls; 1610 cdev->csk_init_act_open = init_act_open; 1611 1612 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1613 return 0; 1614 } 1615 1616 static inline void 1617 ulp_mem_io_set_hdr(struct cxgbi_device *cdev, 1618 struct ulp_mem_io *req, 1619 unsigned int wr_len, unsigned int dlen, 1620 unsigned int pm_addr, 1621 int tid) 1622 { 1623 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1624 struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1); 1625 1626 INIT_ULPTX_WR(req, wr_len, 0, tid); 1627 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_ULPTX_WR) | 1628 FW_WR_ATOMIC_V(0)); 1629 req->cmd = htonl(ULPTX_CMD_V(ULP_TX_MEM_WRITE) | 1630 ULP_MEMIO_ORDER_V(is_t4(lldi->adapter_type)) | 1631 T5_ULP_MEMIO_IMM_V(!is_t4(lldi->adapter_type))); 1632 req->dlen = htonl(ULP_MEMIO_DATA_LEN_V(dlen >> 5)); 1633 req->lock_addr = htonl(ULP_MEMIO_ADDR_V(pm_addr >> 5)); 1634 req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)); 1635 1636 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); 1637 idata->len = htonl(dlen); 1638 } 1639 1640 static struct sk_buff * 1641 ddp_ppod_init_idata(struct cxgbi_device *cdev, 1642 struct cxgbi_ppm *ppm, 1643 unsigned int idx, unsigned int npods, 1644 unsigned int tid) 1645 { 1646 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 1647 unsigned int dlen = npods << PPOD_SIZE_SHIFT; 1648 unsigned int wr_len = roundup(sizeof(struct ulp_mem_io) + 1649 sizeof(struct ulptx_idata) + dlen, 16); 1650 struct sk_buff *skb = alloc_wr(wr_len, 0, GFP_ATOMIC); 1651 1652 if (!skb) { 1653 pr_err("%s: %s idx %u, npods %u, OOM.\n", 1654 __func__, ppm->ndev->name, idx, npods); 1655 return NULL; 1656 } 1657 1658 ulp_mem_io_set_hdr(cdev, (struct ulp_mem_io *)skb->head, wr_len, dlen, 1659 pm_addr, tid); 1660 1661 return skb; 1662 } 1663 1664 static int ddp_ppod_write_idata(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, 1665 struct cxgbi_task_tag_info *ttinfo, 1666 unsigned int idx, unsigned int npods, 1667 struct scatterlist **sg_pp, 1668 unsigned int *sg_off) 1669 { 1670 struct cxgbi_device *cdev = csk->cdev; 1671 struct sk_buff *skb = ddp_ppod_init_idata(cdev, ppm, idx, npods, 1672 csk->tid); 1673 struct ulp_mem_io *req; 1674 struct ulptx_idata *idata; 1675 struct cxgbi_pagepod *ppod; 1676 int i; 1677 1678 if (!skb) 1679 return -ENOMEM; 1680 1681 req = (struct ulp_mem_io *)skb->head; 1682 idata = (struct ulptx_idata *)(req + 1); 1683 ppod = (struct cxgbi_pagepod *)(idata + 1); 1684 1685 for (i = 0; i < npods; i++, ppod++) 1686 cxgbi_ddp_set_one_ppod(ppod, ttinfo, sg_pp, sg_off); 1687 1688 cxgbi_skcb_set_flag(skb, SKCBF_TX_MEM_WRITE); 1689 cxgbi_skcb_set_flag(skb, SKCBF_TX_FLAG_COMPL); 1690 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->port_id); 1691 1692 spin_lock_bh(&csk->lock); 1693 cxgbi_sock_skb_entail(csk, skb); 1694 spin_unlock_bh(&csk->lock); 1695 1696 return 0; 1697 } 1698 1699 static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, 1700 struct cxgbi_task_tag_info *ttinfo) 1701 { 1702 unsigned int pidx = ttinfo->idx; 1703 unsigned int npods = ttinfo->npods; 1704 unsigned int i, cnt; 1705 int err = 0; 1706 struct scatterlist *sg = ttinfo->sgl; 1707 unsigned int offset = 0; 1708 1709 ttinfo->cid = csk->port_id; 1710 1711 for (i = 0; i < npods; i += cnt, pidx += cnt) { 1712 cnt = npods - i; 1713 1714 if (cnt > ULPMEM_IDATA_MAX_NPPODS) 1715 cnt = ULPMEM_IDATA_MAX_NPPODS; 1716 err = ddp_ppod_write_idata(ppm, csk, ttinfo, pidx, cnt, 1717 &sg, &offset); 1718 if (err < 0) 1719 break; 1720 } 1721 1722 return err; 1723 } 1724 1725 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, unsigned int tid, 1726 int pg_idx, bool reply) 1727 { 1728 struct sk_buff *skb; 1729 struct cpl_set_tcb_field *req; 1730 1731 if (!pg_idx || pg_idx >= DDP_PGIDX_MAX) 1732 return 0; 1733 1734 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 1735 if (!skb) 1736 return -ENOMEM; 1737 1738 /* set up ulp page size */ 1739 req = (struct cpl_set_tcb_field *)skb->head; 1740 INIT_TP_WR(req, csk->tid); 1741 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, csk->tid)); 1742 req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid)); 1743 req->word_cookie = htons(0); 1744 req->mask = cpu_to_be64(0x3 << 8); 1745 req->val = cpu_to_be64(pg_idx << 8); 1746 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 1747 1748 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1749 "csk 0x%p, tid 0x%x, pg_idx %u.\n", csk, csk->tid, pg_idx); 1750 1751 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 1752 return 0; 1753 } 1754 1755 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 1756 int hcrc, int dcrc, int reply) 1757 { 1758 struct sk_buff *skb; 1759 struct cpl_set_tcb_field *req; 1760 1761 if (!hcrc && !dcrc) 1762 return 0; 1763 1764 skb = alloc_wr(sizeof(*req), 0, GFP_KERNEL); 1765 if (!skb) 1766 return -ENOMEM; 1767 1768 csk->hcrc_len = (hcrc ? 4 : 0); 1769 csk->dcrc_len = (dcrc ? 4 : 0); 1770 /* set up ulp submode */ 1771 req = (struct cpl_set_tcb_field *)skb->head; 1772 INIT_TP_WR(req, tid); 1773 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1774 req->reply_ctrl = htons(NO_REPLY_V(reply) | QUEUENO_V(csk->rss_qid)); 1775 req->word_cookie = htons(0); 1776 req->mask = cpu_to_be64(0x3 << 4); 1777 req->val = cpu_to_be64(((hcrc ? ULP_CRC_HEADER : 0) | 1778 (dcrc ? ULP_CRC_DATA : 0)) << 4); 1779 set_wr_txq(skb, CPL_PRIORITY_CONTROL, csk->port_id); 1780 1781 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1782 "csk 0x%p, tid 0x%x, crc %d,%d.\n", csk, csk->tid, hcrc, dcrc); 1783 1784 cxgb4_ofld_send(csk->cdev->ports[csk->port_id], skb); 1785 return 0; 1786 } 1787 1788 static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev) 1789 { 1790 return (struct cxgbi_ppm *)(*((struct cxgb4_lld_info *) 1791 (cxgbi_cdev_priv(cdev)))->iscsi_ppm); 1792 } 1793 1794 static int cxgb4i_ddp_init(struct cxgbi_device *cdev) 1795 { 1796 struct cxgb4_lld_info *lldi = cxgbi_cdev_priv(cdev); 1797 struct net_device *ndev = cdev->ports[0]; 1798 struct cxgbi_tag_format tformat; 1799 unsigned int ppmax; 1800 int i; 1801 1802 if (!lldi->vr->iscsi.size) { 1803 pr_warn("%s, iscsi NOT enabled, check config!\n", ndev->name); 1804 return -EACCES; 1805 } 1806 1807 cdev->flags |= CXGBI_FLAG_USE_PPOD_OFLDQ; 1808 ppmax = lldi->vr->iscsi.size >> PPOD_SIZE_SHIFT; 1809 1810 memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); 1811 for (i = 0; i < 4; i++) 1812 tformat.pgsz_order[i] = (lldi->iscsi_pgsz_order >> (i << 3)) 1813 & 0xF; 1814 cxgbi_tagmask_check(lldi->iscsi_tagmask, &tformat); 1815 1816 cxgbi_ddp_ppm_setup(lldi->iscsi_ppm, cdev, &tformat, ppmax, 1817 lldi->iscsi_llimit, lldi->vr->iscsi.start, 2); 1818 1819 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 1820 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 1821 cdev->csk_ddp_set_map = ddp_set_map; 1822 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1823 lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); 1824 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1825 lldi->iscsi_iolen - ISCSI_PDU_NONPAYLOAD_LEN); 1826 cdev->cdev2ppm = cdev2ppm; 1827 1828 return 0; 1829 } 1830 1831 static void *t4_uld_add(const struct cxgb4_lld_info *lldi) 1832 { 1833 struct cxgbi_device *cdev; 1834 struct port_info *pi; 1835 int i, rc; 1836 1837 cdev = cxgbi_device_register(sizeof(*lldi), lldi->nports); 1838 if (!cdev) { 1839 pr_info("t4 device 0x%p, register failed.\n", lldi); 1840 return NULL; 1841 } 1842 pr_info("0x%p,0x%x, ports %u,%s, chan %u, q %u,%u, wr %u.\n", 1843 cdev, lldi->adapter_type, lldi->nports, 1844 lldi->ports[0]->name, lldi->nchan, lldi->ntxq, 1845 lldi->nrxq, lldi->wr_cred); 1846 for (i = 0; i < lldi->nrxq; i++) 1847 log_debug(1 << CXGBI_DBG_DEV, 1848 "t4 0x%p, rxq id #%d: %u.\n", 1849 cdev, i, lldi->rxq_ids[i]); 1850 1851 memcpy(cxgbi_cdev_priv(cdev), lldi, sizeof(*lldi)); 1852 cdev->flags = CXGBI_FLAG_DEV_T4; 1853 cdev->pdev = lldi->pdev; 1854 cdev->ports = lldi->ports; 1855 cdev->nports = lldi->nports; 1856 cdev->mtus = lldi->mtus; 1857 cdev->nmtus = NMTUS; 1858 cdev->rx_credit_thres = (CHELSIO_CHIP_VERSION(lldi->adapter_type) <= 1859 CHELSIO_T5) ? cxgb4i_rx_credit_thres : 0; 1860 cdev->skb_tx_rsvd = CXGB4I_TX_HEADER_LEN; 1861 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr); 1862 cdev->itp = &cxgb4i_iscsi_transport; 1863 1864 cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) 1865 << FW_VIID_PFN_S; 1866 pr_info("cdev 0x%p,%s, pfvf %u.\n", 1867 cdev, lldi->ports[0]->name, cdev->pfvf); 1868 1869 rc = cxgb4i_ddp_init(cdev); 1870 if (rc) { 1871 pr_info("t4 0x%p ddp init failed.\n", cdev); 1872 goto err_out; 1873 } 1874 rc = cxgb4i_ofld_init(cdev); 1875 if (rc) { 1876 pr_info("t4 0x%p ofld init failed.\n", cdev); 1877 goto err_out; 1878 } 1879 1880 rc = cxgbi_hbas_add(cdev, CXGB4I_MAX_LUN, CXGBI_MAX_CONN, 1881 &cxgb4i_host_template, cxgb4i_stt); 1882 if (rc) 1883 goto err_out; 1884 1885 for (i = 0; i < cdev->nports; i++) { 1886 pi = netdev_priv(lldi->ports[i]); 1887 cdev->hbas[i]->port_id = pi->port_id; 1888 } 1889 return cdev; 1890 1891 err_out: 1892 cxgbi_device_unregister(cdev); 1893 return ERR_PTR(-ENOMEM); 1894 } 1895 1896 #define RX_PULL_LEN 128 1897 static int t4_uld_rx_handler(void *handle, const __be64 *rsp, 1898 const struct pkt_gl *pgl) 1899 { 1900 const struct cpl_act_establish *rpl; 1901 struct sk_buff *skb; 1902 unsigned int opc; 1903 struct cxgbi_device *cdev = handle; 1904 1905 if (pgl == NULL) { 1906 unsigned int len = 64 - sizeof(struct rsp_ctrl) - 8; 1907 1908 skb = alloc_wr(len, 0, GFP_ATOMIC); 1909 if (!skb) 1910 goto nomem; 1911 skb_copy_to_linear_data(skb, &rsp[1], len); 1912 } else { 1913 if (unlikely(*(u8 *)rsp != *(u8 *)pgl->va)) { 1914 pr_info("? FL 0x%p,RSS%#llx,FL %#llx,len %u.\n", 1915 pgl->va, be64_to_cpu(*rsp), 1916 be64_to_cpu(*(u64 *)pgl->va), 1917 pgl->tot_len); 1918 return 0; 1919 } 1920 skb = cxgb4_pktgl_to_skb(pgl, RX_PULL_LEN, RX_PULL_LEN); 1921 if (unlikely(!skb)) 1922 goto nomem; 1923 } 1924 1925 rpl = (struct cpl_act_establish *)skb->data; 1926 opc = rpl->ot.opcode; 1927 log_debug(1 << CXGBI_DBG_TOE, 1928 "cdev %p, opcode 0x%x(0x%x,0x%x), skb %p.\n", 1929 cdev, opc, rpl->ot.opcode_tid, ntohl(rpl->ot.opcode_tid), skb); 1930 if (cxgb4i_cplhandlers[opc]) 1931 cxgb4i_cplhandlers[opc](cdev, skb); 1932 else { 1933 pr_err("No handler for opcode 0x%x.\n", opc); 1934 __kfree_skb(skb); 1935 } 1936 return 0; 1937 nomem: 1938 log_debug(1 << CXGBI_DBG_TOE, "OOM bailing out.\n"); 1939 return 1; 1940 } 1941 1942 static int t4_uld_state_change(void *handle, enum cxgb4_state state) 1943 { 1944 struct cxgbi_device *cdev = handle; 1945 1946 switch (state) { 1947 case CXGB4_STATE_UP: 1948 pr_info("cdev 0x%p, UP.\n", cdev); 1949 break; 1950 case CXGB4_STATE_START_RECOVERY: 1951 pr_info("cdev 0x%p, RECOVERY.\n", cdev); 1952 /* close all connections */ 1953 break; 1954 case CXGB4_STATE_DOWN: 1955 pr_info("cdev 0x%p, DOWN.\n", cdev); 1956 break; 1957 case CXGB4_STATE_DETACH: 1958 pr_info("cdev 0x%p, DETACH.\n", cdev); 1959 cxgbi_device_unregister(cdev); 1960 break; 1961 default: 1962 pr_info("cdev 0x%p, unknown state %d.\n", cdev, state); 1963 break; 1964 } 1965 return 0; 1966 } 1967 1968 static int __init cxgb4i_init_module(void) 1969 { 1970 int rc; 1971 1972 printk(KERN_INFO "%s", version); 1973 1974 rc = cxgbi_iscsi_init(&cxgb4i_iscsi_transport, &cxgb4i_stt); 1975 if (rc < 0) 1976 return rc; 1977 cxgb4_register_uld(CXGB4_ULD_ISCSI, &cxgb4i_uld_info); 1978 1979 return 0; 1980 } 1981 1982 static void __exit cxgb4i_exit_module(void) 1983 { 1984 cxgb4_unregister_uld(CXGB4_ULD_ISCSI); 1985 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T4); 1986 cxgbi_iscsi_cleanup(&cxgb4i_iscsi_transport, &cxgb4i_stt); 1987 } 1988 1989 module_init(cxgb4i_init_module); 1990 module_exit(cxgb4i_exit_module); 1991