1 /* 2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management 3 * 4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved. 5 * 6 * This program is distributed in the hope that it will be useful, but WITHOUT 7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 9 * release for licensing terms and conditions. 10 * 11 * Written by: Dimitris Michailidis (dm@chelsio.com) 12 * Karen Xie (kxie@chelsio.com) 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 16 17 #include <linux/version.h> 18 #include <linux/module.h> 19 #include <linux/moduleparam.h> 20 #include <scsi/scsi_host.h> 21 22 #include "common.h" 23 #include "t3_cpl.h" 24 #include "t3cdev.h" 25 #include "cxgb3_defs.h" 26 #include "cxgb3_ctl_defs.h" 27 #include "cxgb3_offload.h" 28 #include "firmware_exports.h" 29 #include "cxgb3i.h" 30 31 static unsigned int dbg_level; 32 #include "../libcxgbi.h" 33 34 #define DRV_MODULE_NAME "cxgb3i" 35 #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver" 36 #define DRV_MODULE_VERSION "2.0.0" 37 #define DRV_MODULE_RELDATE "Jun. 2010" 38 39 static char version[] = 40 DRV_MODULE_DESC " " DRV_MODULE_NAME 41 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 42 43 MODULE_AUTHOR("Chelsio Communications, Inc."); 44 MODULE_DESCRIPTION(DRV_MODULE_DESC); 45 MODULE_VERSION(DRV_MODULE_VERSION); 46 MODULE_LICENSE("GPL"); 47 48 module_param(dbg_level, uint, 0644); 49 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)"); 50 51 static int cxgb3i_rcv_win = 256 * 1024; 52 module_param(cxgb3i_rcv_win, int, 0644); 53 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)"); 54 55 static int cxgb3i_snd_win = 128 * 1024; 56 module_param(cxgb3i_snd_win, int, 0644); 57 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)"); 58 59 static int cxgb3i_rx_credit_thres = 10 * 1024; 60 module_param(cxgb3i_rx_credit_thres, int, 0644); 61 MODULE_PARM_DESC(rx_credit_thres, 62 "RX credits return threshold in bytes (default=10KB)"); 63 64 static unsigned int cxgb3i_max_connect = 8 * 1024; 65 module_param(cxgb3i_max_connect, uint, 0644); 66 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)"); 67 68 static unsigned int cxgb3i_sport_base = 20000; 69 module_param(cxgb3i_sport_base, uint, 0644); 70 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)"); 71 72 static void cxgb3i_dev_open(struct t3cdev *); 73 static void cxgb3i_dev_close(struct t3cdev *); 74 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32); 75 76 static struct cxgb3_client t3_client = { 77 .name = DRV_MODULE_NAME, 78 .handlers = cxgb3i_cpl_handlers, 79 .add = cxgb3i_dev_open, 80 .remove = cxgb3i_dev_close, 81 .event_handler = cxgb3i_dev_event_handler, 82 }; 83 84 static struct scsi_host_template cxgb3i_host_template = { 85 .module = THIS_MODULE, 86 .name = DRV_MODULE_NAME, 87 .proc_name = DRV_MODULE_NAME, 88 .can_queue = CXGB3I_SCSI_HOST_QDEPTH, 89 .queuecommand = iscsi_queuecommand, 90 .change_queue_depth = iscsi_change_queue_depth, 91 .sg_tablesize = SG_ALL, 92 .max_sectors = 0xFFFF, 93 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 94 .eh_abort_handler = iscsi_eh_abort, 95 .eh_device_reset_handler = iscsi_eh_device_reset, 96 .eh_target_reset_handler = iscsi_eh_recover_target, 97 .target_alloc = iscsi_target_alloc, 98 .use_clustering = DISABLE_CLUSTERING, 99 .this_id = -1, 100 }; 101 102 static struct iscsi_transport cxgb3i_iscsi_transport = { 103 .owner = THIS_MODULE, 104 .name = DRV_MODULE_NAME, 105 /* owner and name should be set already */ 106 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 107 | CAP_DATADGST | CAP_DIGEST_OFFLOAD | 108 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, 109 .param_mask = ISCSI_MAX_RECV_DLENGTH | ISCSI_MAX_XMIT_DLENGTH | 110 ISCSI_HDRDGST_EN | ISCSI_DATADGST_EN | 111 ISCSI_INITIAL_R2T_EN | ISCSI_MAX_R2T | 112 ISCSI_IMM_DATA_EN | ISCSI_FIRST_BURST | 113 ISCSI_MAX_BURST | ISCSI_PDU_INORDER_EN | 114 ISCSI_DATASEQ_INORDER_EN | ISCSI_ERL | 115 ISCSI_CONN_PORT | ISCSI_CONN_ADDRESS | 116 ISCSI_EXP_STATSN | ISCSI_PERSISTENT_PORT | 117 ISCSI_PERSISTENT_ADDRESS | 118 ISCSI_TARGET_NAME | ISCSI_TPGT | 119 ISCSI_USERNAME | ISCSI_PASSWORD | 120 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | 121 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | 122 ISCSI_LU_RESET_TMO | ISCSI_TGT_RESET_TMO | 123 ISCSI_PING_TMO | ISCSI_RECV_TMO | 124 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME, 125 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS | 126 ISCSI_HOST_INITIATOR_NAME | 127 ISCSI_HOST_NETDEV_NAME, 128 .get_host_param = cxgbi_get_host_param, 129 .set_host_param = cxgbi_set_host_param, 130 /* session management */ 131 .create_session = cxgbi_create_session, 132 .destroy_session = cxgbi_destroy_session, 133 .get_session_param = iscsi_session_get_param, 134 /* connection management */ 135 .create_conn = cxgbi_create_conn, 136 .bind_conn = cxgbi_bind_conn, 137 .destroy_conn = iscsi_tcp_conn_teardown, 138 .start_conn = iscsi_conn_start, 139 .stop_conn = iscsi_conn_stop, 140 .get_conn_param = iscsi_conn_get_param, 141 .set_param = cxgbi_set_conn_param, 142 .get_stats = cxgbi_get_conn_stats, 143 /* pdu xmit req from user space */ 144 .send_pdu = iscsi_conn_send_pdu, 145 /* task */ 146 .init_task = iscsi_tcp_task_init, 147 .xmit_task = iscsi_tcp_task_xmit, 148 .cleanup_task = cxgbi_cleanup_task, 149 /* pdu */ 150 .alloc_pdu = cxgbi_conn_alloc_pdu, 151 .init_pdu = cxgbi_conn_init_pdu, 152 .xmit_pdu = cxgbi_conn_xmit_pdu, 153 .parse_pdu_itt = cxgbi_parse_pdu_itt, 154 /* TCP connect/disconnect */ 155 .get_ep_param = cxgbi_get_ep_param, 156 .ep_connect = cxgbi_ep_connect, 157 .ep_poll = cxgbi_ep_poll, 158 .ep_disconnect = cxgbi_ep_disconnect, 159 /* Error recovery timeout call */ 160 .session_recovery_timedout = iscsi_session_recovery_timedout, 161 }; 162 163 static struct scsi_transport_template *cxgb3i_stt; 164 165 /* 166 * CPL (Chelsio Protocol Language) defines a message passing interface between 167 * the host driver and Chelsio asic. 168 * The section below implments CPLs that related to iscsi tcp connection 169 * open/close/abort and data send/receive. 170 */ 171 172 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion); 173 174 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 175 const struct l2t_entry *e) 176 { 177 unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win); 178 struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head; 179 180 skb->priority = CPL_PRIORITY_SETUP; 181 182 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 183 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid)); 184 req->local_port = csk->saddr.sin_port; 185 req->peer_port = csk->daddr.sin_port; 186 req->local_ip = csk->saddr.sin_addr.s_addr; 187 req->peer_ip = csk->daddr.sin_addr.s_addr; 188 189 req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS | 190 V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) | 191 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); 192 req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) | 193 V_RCV_BUFSIZ(cxgb3i_rcv_win>>10)); 194 195 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 196 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n", 197 csk, csk->state, csk->flags, csk->atid, 198 &req->local_ip, ntohs(req->local_port), 199 &req->peer_ip, ntohs(req->peer_port), 200 csk->mss_idx, e->idx, e->smt_idx); 201 202 l2t_send(csk->cdev->lldev, skb, csk->l2t); 203 } 204 205 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb) 206 { 207 cxgbi_sock_act_open_req_arp_failure(NULL, skb); 208 } 209 210 /* 211 * CPL connection close request: host -> 212 * 213 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to 214 * the write queue (i.e., after any unsent txt data). 215 */ 216 static void send_close_req(struct cxgbi_sock *csk) 217 { 218 struct sk_buff *skb = csk->cpl_close; 219 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 220 unsigned int tid = csk->tid; 221 222 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 223 "csk 0x%p,%u,0x%lx,%u.\n", 224 csk, csk->state, csk->flags, csk->tid); 225 226 csk->cpl_close = NULL; 227 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 228 req->wr.wr_lo = htonl(V_WR_TID(tid)); 229 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 230 req->rsvd = htonl(csk->write_seq); 231 232 cxgbi_sock_skb_entail(csk, skb); 233 if (csk->state >= CTP_ESTABLISHED) 234 push_tx_frames(csk, 1); 235 } 236 237 /* 238 * CPL connection abort request: host -> 239 * 240 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs 241 * for the same connection and also that we do not try to send a message 242 * after the connection has closed. 243 */ 244 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb) 245 { 246 struct cpl_abort_req *req = cplhdr(skb); 247 248 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 249 "t3dev 0x%p, tid %u, skb 0x%p.\n", 250 tdev, GET_TID(req), skb); 251 req->cmd = CPL_ABORT_NO_RST; 252 cxgb3_ofld_send(tdev, skb); 253 } 254 255 static void send_abort_req(struct cxgbi_sock *csk) 256 { 257 struct sk_buff *skb = csk->cpl_abort_req; 258 struct cpl_abort_req *req; 259 260 if (unlikely(csk->state == CTP_ABORTING || !skb)) 261 return; 262 cxgbi_sock_set_state(csk, CTP_ABORTING); 263 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 264 /* Purge the send queue so we don't send anything after an abort. */ 265 cxgbi_sock_purge_write_queue(csk); 266 267 csk->cpl_abort_req = NULL; 268 req = (struct cpl_abort_req *)skb->head; 269 skb->priority = CPL_PRIORITY_DATA; 270 set_arp_failure_handler(skb, abort_arp_failure); 271 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 272 req->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 273 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 274 req->rsvd0 = htonl(csk->snd_nxt); 275 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 276 req->cmd = CPL_ABORT_SEND_RST; 277 278 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 279 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 280 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 281 req->rsvd1); 282 283 l2t_send(csk->cdev->lldev, skb, csk->l2t); 284 } 285 286 /* 287 * CPL connection abort reply: host -> 288 * 289 * Send an ABORT_RPL message in response of the ABORT_REQ received. 290 */ 291 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 292 { 293 struct sk_buff *skb = csk->cpl_abort_rpl; 294 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 295 296 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 297 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 298 csk, csk->state, csk->flags, csk->tid, rst_status); 299 300 csk->cpl_abort_rpl = NULL; 301 skb->priority = CPL_PRIORITY_DATA; 302 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 303 rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 304 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 305 rpl->cmd = rst_status; 306 cxgb3_ofld_send(csk->cdev->lldev, skb); 307 } 308 309 /* 310 * CPL connection rx data ack: host -> 311 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 312 * credits sent. 313 */ 314 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 315 { 316 struct sk_buff *skb; 317 struct cpl_rx_data_ack *req; 318 u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 319 320 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 321 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n", 322 csk, csk->state, csk->flags, csk->tid, credits, dack); 323 324 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 325 if (!skb) { 326 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 327 return 0; 328 } 329 req = (struct cpl_rx_data_ack *)skb->head; 330 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 331 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid)); 332 req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) | 333 V_RX_CREDITS(credits)); 334 skb->priority = CPL_PRIORITY_ACK; 335 cxgb3_ofld_send(csk->cdev->lldev, skb); 336 return credits; 337 } 338 339 /* 340 * CPL connection tx data: host -> 341 * 342 * Send iscsi PDU via TX_DATA CPL message. Returns the number of 343 * credits sent. 344 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of 345 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3). 346 */ 347 348 static unsigned int wrlen __read_mostly; 349 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly; 350 351 static void init_wr_tab(unsigned int wr_len) 352 { 353 int i; 354 355 if (skb_wrs[1]) /* already initialized */ 356 return; 357 for (i = 1; i < SKB_WR_LIST_SIZE; i++) { 358 int sgl_len = (3 * i) / 2 + (i & 1); 359 360 sgl_len += 3; 361 skb_wrs[i] = (sgl_len <= wr_len 362 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1)); 363 } 364 wrlen = wr_len * 8; 365 } 366 367 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 368 int len, int req_completion) 369 { 370 struct tx_data_wr *req; 371 struct l2t_entry *l2t = csk->l2t; 372 373 skb_reset_transport_header(skb); 374 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req)); 375 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) | 376 (req_completion ? F_WR_COMPL : 0)); 377 req->wr_lo = htonl(V_WR_TID(csk->tid)); 378 /* len includes the length of any HW ULP additions */ 379 req->len = htonl(len); 380 /* V_TX_ULP_SUBMODE sets both the mode and submode */ 381 req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) | 382 V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1))); 383 req->sndseq = htonl(csk->snd_nxt); 384 req->param = htonl(V_TX_PORT(l2t->smt_idx)); 385 386 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 387 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | 388 V_TX_CPU_IDX(csk->rss_qid)); 389 /* sendbuffer is in units of 32KB. */ 390 req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15)); 391 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 392 } 393 } 394 395 /** 396 * push_tx_frames -- start transmit 397 * @c3cn: the offloaded connection 398 * @req_completion: request wr_ack or not 399 * 400 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a 401 * connection's send queue and sends them on to T3. Must be called with the 402 * connection's lock held. Returns the amount of send buffer space that was 403 * freed as a result of sending queued data to T3. 404 */ 405 406 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb) 407 { 408 kfree_skb(skb); 409 } 410 411 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 412 { 413 int total_size = 0; 414 struct sk_buff *skb; 415 416 if (unlikely(csk->state < CTP_ESTABLISHED || 417 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 418 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 419 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 420 csk, csk->state, csk->flags, csk->tid); 421 return 0; 422 } 423 424 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { 425 int len = skb->len; /* length before skb_push */ 426 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len); 427 int wrs_needed = skb_wrs[frags]; 428 429 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen) 430 wrs_needed = 1; 431 432 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1); 433 434 if (csk->wr_cred < wrs_needed) { 435 log_debug(1 << CXGBI_DBG_PDU_TX, 436 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n", 437 csk, skb->len, skb->data_len, frags, 438 wrs_needed, csk->wr_cred); 439 break; 440 } 441 442 __skb_unlink(skb, &csk->write_queue); 443 skb->priority = CPL_PRIORITY_DATA; 444 skb->csum = wrs_needed; /* remember this until the WR_ACK */ 445 csk->wr_cred -= wrs_needed; 446 csk->wr_una_cred += wrs_needed; 447 cxgbi_sock_enqueue_wr(csk, skb); 448 449 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 450 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, " 451 "left %u, unack %u.\n", 452 csk, skb->len, skb->data_len, frags, skb->csum, 453 csk->wr_cred, csk->wr_una_cred); 454 455 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 456 if ((req_completion && 457 csk->wr_una_cred == wrs_needed) || 458 csk->wr_una_cred >= csk->wr_max_cred / 2) { 459 req_completion = 1; 460 csk->wr_una_cred = 0; 461 } 462 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 463 make_tx_data_wr(csk, skb, len, req_completion); 464 csk->snd_nxt += len; 465 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 466 } 467 total_size += skb->truesize; 468 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 469 "csk 0x%p, tid 0x%x, send skb 0x%p.\n", 470 csk, csk->tid, skb); 471 set_arp_failure_handler(skb, arp_failure_skb_discard); 472 l2t_send(csk->cdev->lldev, skb, csk->l2t); 473 } 474 return total_size; 475 } 476 477 /* 478 * Process a CPL_ACT_ESTABLISH message: -> host 479 * Updates connection state from an active establish CPL message. Runs with 480 * the connection lock held. 481 */ 482 483 static inline void free_atid(struct cxgbi_sock *csk) 484 { 485 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 486 cxgb3_free_atid(csk->cdev->lldev, csk->atid); 487 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 488 cxgbi_sock_put(csk); 489 } 490 } 491 492 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 493 { 494 struct cxgbi_sock *csk = ctx; 495 struct cpl_act_establish *req = cplhdr(skb); 496 unsigned int tid = GET_TID(req); 497 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 498 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ 499 500 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 501 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 502 atid, atid, csk, csk->state, csk->flags, rcv_isn); 503 504 cxgbi_sock_get(csk); 505 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 506 csk->tid = tid; 507 cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid); 508 509 free_atid(csk); 510 511 csk->rss_qid = G_QNUM(ntohs(skb->csum)); 512 513 spin_lock_bh(&csk->lock); 514 if (csk->retry_timer.function) { 515 del_timer(&csk->retry_timer); 516 csk->retry_timer.function = NULL; 517 } 518 519 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 520 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 521 csk, csk->state, csk->flags, csk->tid); 522 523 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 524 if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10)) 525 csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10); 526 527 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 528 529 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 530 /* upper layer has requested closing */ 531 send_abort_req(csk); 532 else { 533 if (skb_queue_len(&csk->write_queue)) 534 push_tx_frames(csk, 1); 535 cxgbi_conn_tx_open(csk); 536 } 537 538 spin_unlock_bh(&csk->lock); 539 __kfree_skb(skb); 540 return 0; 541 } 542 543 /* 544 * Process a CPL_ACT_OPEN_RPL message: -> host 545 * Handle active open failures. 546 */ 547 static int act_open_rpl_status_to_errno(int status) 548 { 549 switch (status) { 550 case CPL_ERR_CONN_RESET: 551 return -ECONNREFUSED; 552 case CPL_ERR_ARP_MISS: 553 return -EHOSTUNREACH; 554 case CPL_ERR_CONN_TIMEDOUT: 555 return -ETIMEDOUT; 556 case CPL_ERR_TCAM_FULL: 557 return -ENOMEM; 558 case CPL_ERR_CONN_EXIST: 559 return -EADDRINUSE; 560 default: 561 return -EIO; 562 } 563 } 564 565 static void act_open_retry_timer(unsigned long data) 566 { 567 struct sk_buff *skb; 568 struct cxgbi_sock *csk = (struct cxgbi_sock *)data; 569 570 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 571 "csk 0x%p,%u,0x%lx,%u.\n", 572 csk, csk->state, csk->flags, csk->tid); 573 574 cxgbi_sock_get(csk); 575 spin_lock_bh(&csk->lock); 576 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC); 577 if (!skb) 578 cxgbi_sock_fail_act_open(csk, -ENOMEM); 579 else { 580 skb->sk = (struct sock *)csk; 581 set_arp_failure_handler(skb, act_open_arp_failure); 582 send_act_open_req(csk, skb, csk->l2t); 583 } 584 spin_unlock_bh(&csk->lock); 585 cxgbi_sock_put(csk); 586 } 587 588 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 589 { 590 struct cxgbi_sock *csk = ctx; 591 struct cpl_act_open_rpl *rpl = cplhdr(skb); 592 593 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n", 594 csk, csk->state, csk->flags, csk->atid, rpl->status, 595 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 596 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 597 598 if (rpl->status != CPL_ERR_TCAM_FULL && 599 rpl->status != CPL_ERR_CONN_EXIST && 600 rpl->status != CPL_ERR_ARP_MISS) 601 cxgb3_queue_tid_release(tdev, GET_TID(rpl)); 602 603 cxgbi_sock_get(csk); 604 spin_lock_bh(&csk->lock); 605 if (rpl->status == CPL_ERR_CONN_EXIST && 606 csk->retry_timer.function != act_open_retry_timer) { 607 csk->retry_timer.function = act_open_retry_timer; 608 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 609 } else 610 cxgbi_sock_fail_act_open(csk, 611 act_open_rpl_status_to_errno(rpl->status)); 612 613 spin_unlock_bh(&csk->lock); 614 cxgbi_sock_put(csk); 615 __kfree_skb(skb); 616 return 0; 617 } 618 619 /* 620 * Process PEER_CLOSE CPL messages: -> host 621 * Handle peer FIN. 622 */ 623 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 624 { 625 struct cxgbi_sock *csk = ctx; 626 627 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 628 "csk 0x%p,%u,0x%lx,%u.\n", 629 csk, csk->state, csk->flags, csk->tid); 630 631 cxgbi_sock_rcv_peer_close(csk); 632 __kfree_skb(skb); 633 return 0; 634 } 635 636 /* 637 * Process CLOSE_CONN_RPL CPL message: -> host 638 * Process a peer ACK to our FIN. 639 */ 640 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb, 641 void *ctx) 642 { 643 struct cxgbi_sock *csk = ctx; 644 struct cpl_close_con_rpl *rpl = cplhdr(skb); 645 646 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 647 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n", 648 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt)); 649 650 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 651 __kfree_skb(skb); 652 return 0; 653 } 654 655 /* 656 * Process ABORT_REQ_RSS CPL message: -> host 657 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this 658 * request except that we need to reply to it. 659 */ 660 661 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 662 int *need_rst) 663 { 664 switch (abort_reason) { 665 case CPL_ERR_BAD_SYN: /* fall through */ 666 case CPL_ERR_CONN_RESET: 667 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET; 668 case CPL_ERR_XMIT_TIMEDOUT: 669 case CPL_ERR_PERSIST_TIMEDOUT: 670 case CPL_ERR_FINWAIT2_TIMEDOUT: 671 case CPL_ERR_KEEPALIVE_TIMEDOUT: 672 return -ETIMEDOUT; 673 default: 674 return -EIO; 675 } 676 } 677 678 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 679 { 680 const struct cpl_abort_req_rss *req = cplhdr(skb); 681 struct cxgbi_sock *csk = ctx; 682 int rst_status = CPL_ABORT_NO_RST; 683 684 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 685 "csk 0x%p,%u,0x%lx,%u.\n", 686 csk, csk->state, csk->flags, csk->tid); 687 688 if (req->status == CPL_ERR_RTX_NEG_ADVICE || 689 req->status == CPL_ERR_PERSIST_NEG_ADVICE) { 690 goto done; 691 } 692 693 cxgbi_sock_get(csk); 694 spin_lock_bh(&csk->lock); 695 696 if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) { 697 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 698 cxgbi_sock_set_state(csk, CTP_ABORTING); 699 goto out; 700 } 701 702 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 703 send_abort_rpl(csk, rst_status); 704 705 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 706 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 707 cxgbi_sock_closed(csk); 708 } 709 710 out: 711 spin_unlock_bh(&csk->lock); 712 cxgbi_sock_put(csk); 713 done: 714 __kfree_skb(skb); 715 return 0; 716 } 717 718 /* 719 * Process ABORT_RPL_RSS CPL message: -> host 720 * Process abort replies. We only process these messages if we anticipate 721 * them as the coordination between SW and HW in this area is somewhat lacking 722 * and sometimes we get ABORT_RPLs after we are done with the connection that 723 * originated the ABORT_REQ. 724 */ 725 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 726 { 727 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 728 struct cxgbi_sock *csk = ctx; 729 730 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 731 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n", 732 rpl->status, csk, csk ? csk->state : 0, 733 csk ? csk->flags : 0UL); 734 /* 735 * Ignore replies to post-close aborts indicating that the abort was 736 * requested too late. These connections are terminated when we get 737 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss 738 * arrives the TID is either no longer used or it has been recycled. 739 */ 740 if (rpl->status == CPL_ERR_ABORT_FAILED) 741 goto rel_skb; 742 /* 743 * Sometimes we've already closed the connection, e.g., a post-close 744 * abort races with ABORT_REQ_RSS, the latter frees the connection 745 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, 746 * but FW turns the ABORT_REQ into a regular one and so we get 747 * ABORT_RPL_RSS with status 0 and no connection. 748 */ 749 if (csk) 750 cxgbi_sock_rcv_abort_rpl(csk); 751 rel_skb: 752 __kfree_skb(skb); 753 return 0; 754 } 755 756 /* 757 * Process RX_ISCSI_HDR CPL message: -> host 758 * Handle received PDUs, the payload could be DDP'ed. If not, the payload 759 * follow after the bhs. 760 */ 761 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx) 762 { 763 struct cxgbi_sock *csk = ctx; 764 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb); 765 struct cpl_iscsi_hdr_norss data_cpl; 766 struct cpl_rx_data_ddp_norss ddp_cpl; 767 unsigned int hdr_len, data_len, status; 768 unsigned int len; 769 int err; 770 771 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 772 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n", 773 csk, csk->state, csk->flags, csk->tid, skb, skb->len); 774 775 spin_lock_bh(&csk->lock); 776 777 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 778 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 779 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 780 csk, csk->state, csk->flags, csk->tid); 781 if (csk->state != CTP_ABORTING) 782 goto abort_conn; 783 else 784 goto discard; 785 } 786 787 cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq); 788 cxgbi_skcb_flags(skb) = 0; 789 790 skb_reset_transport_header(skb); 791 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr)); 792 793 len = hdr_len = ntohs(hdr_cpl->len); 794 /* msg coalesce is off or not enough data received */ 795 if (skb->len <= hdr_len) { 796 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n", 797 csk->cdev->ports[csk->port_id]->name, csk->tid, 798 skb->len, hdr_len); 799 goto abort_conn; 800 } 801 cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED); 802 803 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl, 804 sizeof(ddp_cpl)); 805 if (err < 0) { 806 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n", 807 csk->cdev->ports[csk->port_id]->name, csk->tid, 808 skb->len, sizeof(ddp_cpl), err); 809 goto abort_conn; 810 } 811 812 cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); 813 cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len); 814 cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc); 815 status = ntohl(ddp_cpl.ddp_status); 816 817 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 818 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n", 819 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status); 820 821 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) 822 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); 823 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) 824 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); 825 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) 826 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); 827 828 if (skb->len > (hdr_len + sizeof(ddp_cpl))) { 829 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl)); 830 if (err < 0) { 831 pr_err("%s: tid %u, cp %zu/%u failed %d.\n", 832 csk->cdev->ports[csk->port_id]->name, 833 csk->tid, sizeof(data_cpl), skb->len, err); 834 goto abort_conn; 835 } 836 data_len = ntohs(data_cpl.len); 837 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX, 838 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n", 839 skb, data_len, cxgbi_skcb_rx_pdulen(skb), status); 840 len += sizeof(data_cpl) + data_len; 841 } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) 842 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); 843 844 csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb); 845 __pskb_trim(skb, len); 846 __skb_queue_tail(&csk->receive_queue, skb); 847 cxgbi_conn_pdu_ready(csk); 848 849 spin_unlock_bh(&csk->lock); 850 return 0; 851 852 abort_conn: 853 send_abort_req(csk); 854 discard: 855 spin_unlock_bh(&csk->lock); 856 __kfree_skb(skb); 857 return 0; 858 } 859 860 /* 861 * Process TX_DATA_ACK CPL messages: -> host 862 * Process an acknowledgment of WR completion. Advance snd_una and send the 863 * next batch of work requests from the write queue. 864 */ 865 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 866 { 867 struct cxgbi_sock *csk = ctx; 868 struct cpl_wr_ack *hdr = cplhdr(skb); 869 870 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 871 "csk 0x%p,%u,0x%lx,%u, cr %u.\n", 872 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits)); 873 874 cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1); 875 __kfree_skb(skb); 876 return 0; 877 } 878 879 /* 880 * for each connection, pre-allocate skbs needed for close/abort requests. So 881 * that we can service the request right away. 882 */ 883 static int alloc_cpls(struct cxgbi_sock *csk) 884 { 885 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0, 886 GFP_KERNEL); 887 if (!csk->cpl_close) 888 return -ENOMEM; 889 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0, 890 GFP_KERNEL); 891 if (!csk->cpl_abort_req) 892 goto free_cpl_skbs; 893 894 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0, 895 GFP_KERNEL); 896 if (!csk->cpl_abort_rpl) 897 goto free_cpl_skbs; 898 899 return 0; 900 901 free_cpl_skbs: 902 cxgbi_sock_free_cpl_skbs(csk); 903 return -ENOMEM; 904 } 905 906 /** 907 * release_offload_resources - release offload resource 908 * @c3cn: the offloaded iscsi tcp connection. 909 * Release resources held by an offload connection (TID, L2T entry, etc.) 910 */ 911 static void l2t_put(struct cxgbi_sock *csk) 912 { 913 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 914 915 if (csk->l2t) { 916 l2t_release(L2DATA(t3dev), csk->l2t); 917 csk->l2t = NULL; 918 cxgbi_sock_put(csk); 919 } 920 } 921 922 static void release_offload_resources(struct cxgbi_sock *csk) 923 { 924 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 925 926 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 927 "csk 0x%p,%u,0x%lx,%u.\n", 928 csk, csk->state, csk->flags, csk->tid); 929 930 csk->rss_qid = 0; 931 cxgbi_sock_free_cpl_skbs(csk); 932 933 if (csk->wr_cred != csk->wr_max_cred) { 934 cxgbi_sock_purge_wr_queue(csk); 935 cxgbi_sock_reset_wr_list(csk); 936 } 937 l2t_put(csk); 938 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 939 free_atid(csk); 940 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 941 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid); 942 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 943 cxgbi_sock_put(csk); 944 } 945 csk->dst = NULL; 946 csk->cdev = NULL; 947 } 948 949 static void update_address(struct cxgbi_hba *chba) 950 { 951 if (chba->ipv4addr) { 952 if (chba->vdev && 953 chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) { 954 cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr); 955 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 956 pr_info("%s set %pI4.\n", 957 chba->vdev->name, &chba->ipv4addr); 958 } else if (chba->ipv4addr != 959 cxgb3i_get_private_ipv4addr(chba->ndev)) { 960 cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr); 961 pr_info("%s set %pI4.\n", 962 chba->ndev->name, &chba->ipv4addr); 963 } 964 } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) { 965 if (chba->vdev) 966 cxgb3i_set_private_ipv4addr(chba->vdev, 0); 967 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 968 } 969 } 970 971 static int init_act_open(struct cxgbi_sock *csk) 972 { 973 struct dst_entry *dst = csk->dst; 974 struct cxgbi_device *cdev = csk->cdev; 975 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 976 struct net_device *ndev = cdev->ports[csk->port_id]; 977 struct cxgbi_hba *chba = cdev->hbas[csk->port_id]; 978 struct sk_buff *skb = NULL; 979 980 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 981 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags); 982 983 update_address(chba); 984 if (chba->ipv4addr) 985 csk->saddr.sin_addr.s_addr = chba->ipv4addr; 986 987 csk->rss_qid = 0; 988 csk->l2t = t3_l2t_get(t3dev, dst_get_neighbour(dst), ndev); 989 if (!csk->l2t) { 990 pr_err("NO l2t available.\n"); 991 return -EINVAL; 992 } 993 cxgbi_sock_get(csk); 994 995 csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk); 996 if (csk->atid < 0) { 997 pr_err("NO atid available.\n"); 998 goto rel_resource; 999 } 1000 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 1001 cxgbi_sock_get(csk); 1002 1003 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL); 1004 if (!skb) 1005 goto rel_resource; 1006 skb->sk = (struct sock *)csk; 1007 set_arp_failure_handler(skb, act_open_arp_failure); 1008 1009 csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1; 1010 csk->wr_una_cred = 0; 1011 csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst)); 1012 cxgbi_sock_reset_wr_list(csk); 1013 csk->err = 0; 1014 1015 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1016 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n", 1017 csk, csk->state, csk->flags, 1018 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 1019 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 1020 1021 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1022 send_act_open_req(csk, skb, csk->l2t); 1023 return 0; 1024 1025 rel_resource: 1026 if (skb) 1027 __kfree_skb(skb); 1028 return -EINVAL; 1029 } 1030 1031 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = { 1032 [CPL_ACT_ESTABLISH] = do_act_establish, 1033 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1034 [CPL_PEER_CLOSE] = do_peer_close, 1035 [CPL_ABORT_REQ_RSS] = do_abort_req, 1036 [CPL_ABORT_RPL_RSS] = do_abort_rpl, 1037 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1038 [CPL_TX_DMA_ACK] = do_wr_ack, 1039 [CPL_ISCSI_HDR] = do_iscsi_hdr, 1040 }; 1041 1042 /** 1043 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found 1044 * @cdev: cxgbi adapter 1045 */ 1046 int cxgb3i_ofld_init(struct cxgbi_device *cdev) 1047 { 1048 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 1049 struct adap_ports port; 1050 struct ofld_page_info rx_page_info; 1051 unsigned int wr_len; 1052 int rc; 1053 1054 if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 || 1055 t3dev->ctl(t3dev, GET_PORTS, &port) < 0 || 1056 t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) { 1057 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev); 1058 return -EINVAL; 1059 } 1060 1061 if (cxgb3i_max_connect > CXGBI_MAX_CONN) 1062 cxgb3i_max_connect = CXGBI_MAX_CONN; 1063 1064 rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base, 1065 cxgb3i_max_connect); 1066 if (rc < 0) 1067 return rc; 1068 1069 init_wr_tab(wr_len); 1070 cdev->csk_release_offload_resources = release_offload_resources; 1071 cdev->csk_push_tx_frames = push_tx_frames; 1072 cdev->csk_send_abort_req = send_abort_req; 1073 cdev->csk_send_close_req = send_close_req; 1074 cdev->csk_send_rx_credits = send_rx_credits; 1075 cdev->csk_alloc_cpls = alloc_cpls; 1076 cdev->csk_init_act_open = init_act_open; 1077 1078 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1079 return 0; 1080 } 1081 1082 /* 1083 * functions to program the pagepod in h/w 1084 */ 1085 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr) 1086 { 1087 struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head; 1088 1089 memset(req, 0, sizeof(*req)); 1090 1091 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 1092 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) | 1093 V_ULPTX_CMD(ULP_MEM_WRITE)); 1094 req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) | 1095 V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1)); 1096 } 1097 1098 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr, 1099 unsigned int idx, unsigned int npods, 1100 struct cxgbi_gather_list *gl) 1101 { 1102 struct cxgbi_device *cdev = csk->cdev; 1103 struct cxgbi_ddp_info *ddp = cdev->ddp; 1104 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit; 1105 int i; 1106 1107 log_debug(1 << CXGBI_DBG_DDP, 1108 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n", 1109 csk, idx, npods, gl); 1110 1111 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) { 1112 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + 1113 PPOD_SIZE, 0, GFP_ATOMIC); 1114 1115 if (!skb) 1116 return -ENOMEM; 1117 1118 ulp_mem_io_set_hdr(skb, pm_addr); 1119 cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head + 1120 sizeof(struct ulp_mem_io)), 1121 hdr, gl, i * PPOD_PAGES_MAX); 1122 skb->priority = CPL_PRIORITY_CONTROL; 1123 cxgb3_ofld_send(cdev->lldev, skb); 1124 } 1125 return 0; 1126 } 1127 1128 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag, 1129 unsigned int idx, unsigned int npods) 1130 { 1131 struct cxgbi_device *cdev = chba->cdev; 1132 struct cxgbi_ddp_info *ddp = cdev->ddp; 1133 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit; 1134 int i; 1135 1136 log_debug(1 << CXGBI_DBG_DDP, 1137 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n", 1138 cdev, idx, npods, tag); 1139 1140 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) { 1141 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + 1142 PPOD_SIZE, 0, GFP_ATOMIC); 1143 1144 if (!skb) { 1145 pr_err("tag 0x%x, 0x%x, %d/%u, skb OOM.\n", 1146 tag, idx, i, npods); 1147 continue; 1148 } 1149 ulp_mem_io_set_hdr(skb, pm_addr); 1150 skb->priority = CPL_PRIORITY_CONTROL; 1151 cxgb3_ofld_send(cdev->lldev, skb); 1152 } 1153 } 1154 1155 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, 1156 unsigned int tid, int pg_idx, bool reply) 1157 { 1158 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1159 GFP_KERNEL); 1160 struct cpl_set_tcb_field *req; 1161 u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0; 1162 1163 log_debug(1 << CXGBI_DBG_DDP, 1164 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx); 1165 if (!skb) 1166 return -ENOMEM; 1167 1168 /* set up ulp submode and page size */ 1169 req = (struct cpl_set_tcb_field *)skb->head; 1170 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1171 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1172 req->reply = V_NO_REPLY(reply ? 0 : 1); 1173 req->cpu_idx = 0; 1174 req->word = htons(31); 1175 req->mask = cpu_to_be64(0xF0000000); 1176 req->val = cpu_to_be64(val << 28); 1177 skb->priority = CPL_PRIORITY_CONTROL; 1178 1179 cxgb3_ofld_send(csk->cdev->lldev, skb); 1180 return 0; 1181 } 1182 1183 /** 1184 * cxgb3i_setup_conn_digest - setup conn. digest setting 1185 * @csk: cxgb tcp socket 1186 * @tid: connection id 1187 * @hcrc: header digest enabled 1188 * @dcrc: data digest enabled 1189 * @reply: request reply from h/w 1190 * set up the iscsi digest settings for a connection identified by tid 1191 */ 1192 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 1193 int hcrc, int dcrc, int reply) 1194 { 1195 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1196 GFP_KERNEL); 1197 struct cpl_set_tcb_field *req; 1198 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0); 1199 1200 log_debug(1 << CXGBI_DBG_DDP, 1201 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc); 1202 if (!skb) 1203 return -ENOMEM; 1204 1205 /* set up ulp submode and page size */ 1206 req = (struct cpl_set_tcb_field *)skb->head; 1207 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1208 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1209 req->reply = V_NO_REPLY(reply ? 0 : 1); 1210 req->cpu_idx = 0; 1211 req->word = htons(31); 1212 req->mask = cpu_to_be64(0x0F000000); 1213 req->val = cpu_to_be64(val << 24); 1214 skb->priority = CPL_PRIORITY_CONTROL; 1215 1216 cxgb3_ofld_send(csk->cdev->lldev, skb); 1217 return 0; 1218 } 1219 1220 /** 1221 * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource 1222 * @cdev: cxgb3i adapter 1223 * release all the resource held by the ddp pagepod manager for a given 1224 * adapter if needed 1225 */ 1226 1227 static void t3_ddp_cleanup(struct cxgbi_device *cdev) 1228 { 1229 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; 1230 1231 if (cxgbi_ddp_cleanup(cdev)) { 1232 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev); 1233 tdev->ulp_iscsi = NULL; 1234 } 1235 } 1236 1237 /** 1238 * ddp_init - initialize the cxgb3 adapter's ddp resource 1239 * @cdev: cxgb3i adapter 1240 * initialize the ddp pagepod manager for a given adapter 1241 */ 1242 static int cxgb3i_ddp_init(struct cxgbi_device *cdev) 1243 { 1244 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; 1245 struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi; 1246 struct ulp_iscsi_info uinfo; 1247 unsigned int pgsz_factor[4]; 1248 int i, err; 1249 1250 if (ddp) { 1251 kref_get(&ddp->refcnt); 1252 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n", 1253 tdev, tdev->ulp_iscsi); 1254 cdev->ddp = ddp; 1255 return -EALREADY; 1256 } 1257 1258 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo); 1259 if (err < 0) { 1260 pr_err("%s, failed to get iscsi param err=%d.\n", 1261 tdev->name, err); 1262 return err; 1263 } 1264 1265 err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit, 1266 uinfo.max_txsz, uinfo.max_rxsz); 1267 if (err < 0) 1268 return err; 1269 1270 ddp = cdev->ddp; 1271 1272 uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT; 1273 cxgbi_ddp_page_size_factor(pgsz_factor); 1274 for (i = 0; i < 4; i++) 1275 uinfo.pgsz_factor[i] = pgsz_factor[i]; 1276 uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT); 1277 1278 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo); 1279 if (err < 0) { 1280 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n", 1281 tdev->name, err); 1282 cxgbi_ddp_cleanup(cdev); 1283 return err; 1284 } 1285 tdev->ulp_iscsi = ddp; 1286 1287 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 1288 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 1289 cdev->csk_ddp_set = ddp_set_map; 1290 cdev->csk_ddp_clear = ddp_clear_map; 1291 1292 pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, " 1293 "%u/%u.\n", 1294 tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask, 1295 ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz, 1296 ddp->max_rxsz, uinfo.max_rxsz); 1297 return 0; 1298 } 1299 1300 static void cxgb3i_dev_close(struct t3cdev *t3dev) 1301 { 1302 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1303 1304 if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) { 1305 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0); 1306 return; 1307 } 1308 1309 cxgbi_device_unregister(cdev); 1310 } 1311 1312 /** 1313 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings 1314 * @t3dev: t3cdev adapter 1315 */ 1316 static void cxgb3i_dev_open(struct t3cdev *t3dev) 1317 { 1318 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1319 struct adapter *adapter = tdev2adap(t3dev); 1320 int i, err; 1321 1322 if (cdev) { 1323 pr_info("0x%p, updating.\n", cdev); 1324 return; 1325 } 1326 1327 cdev = cxgbi_device_register(0, adapter->params.nports); 1328 if (!cdev) { 1329 pr_warn("device 0x%p register failed.\n", t3dev); 1330 return; 1331 } 1332 1333 cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET; 1334 cdev->lldev = t3dev; 1335 cdev->pdev = adapter->pdev; 1336 cdev->ports = adapter->port; 1337 cdev->nports = adapter->params.nports; 1338 cdev->mtus = adapter->params.mtus; 1339 cdev->nmtus = NMTUS; 1340 cdev->snd_win = cxgb3i_snd_win; 1341 cdev->rcv_win = cxgb3i_rcv_win; 1342 cdev->rx_credit_thres = cxgb3i_rx_credit_thres; 1343 cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN; 1344 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss); 1345 cdev->dev_ddp_cleanup = t3_ddp_cleanup; 1346 cdev->itp = &cxgb3i_iscsi_transport; 1347 1348 err = cxgb3i_ddp_init(cdev); 1349 if (err) { 1350 pr_info("0x%p ddp init failed\n", cdev); 1351 goto err_out; 1352 } 1353 1354 err = cxgb3i_ofld_init(cdev); 1355 if (err) { 1356 pr_info("0x%p offload init failed\n", cdev); 1357 goto err_out; 1358 } 1359 1360 err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN, 1361 &cxgb3i_host_template, cxgb3i_stt); 1362 if (err) 1363 goto err_out; 1364 1365 for (i = 0; i < cdev->nports; i++) 1366 cdev->hbas[i]->ipv4addr = 1367 cxgb3i_get_private_ipv4addr(cdev->ports[i]); 1368 1369 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n", 1370 cdev, cdev ? cdev->flags : 0, t3dev, err); 1371 return; 1372 1373 err_out: 1374 cxgbi_device_unregister(cdev); 1375 } 1376 1377 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port) 1378 { 1379 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1380 1381 log_debug(1 << CXGBI_DBG_TOE, 1382 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n", 1383 t3dev, cdev, event, port); 1384 if (!cdev) 1385 return; 1386 1387 switch (event) { 1388 case OFFLOAD_STATUS_DOWN: 1389 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET; 1390 break; 1391 case OFFLOAD_STATUS_UP: 1392 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET; 1393 break; 1394 } 1395 } 1396 1397 /** 1398 * cxgb3i_init_module - module init entry point 1399 * 1400 * initialize any driver wide global data structures and register itself 1401 * with the cxgb3 module 1402 */ 1403 static int __init cxgb3i_init_module(void) 1404 { 1405 int rc; 1406 1407 printk(KERN_INFO "%s", version); 1408 1409 rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1410 if (rc < 0) 1411 return rc; 1412 1413 cxgb3_register_client(&t3_client); 1414 return 0; 1415 } 1416 1417 /** 1418 * cxgb3i_exit_module - module cleanup/exit entry point 1419 * 1420 * go through the driver hba list and for each hba, release any resource held. 1421 * and unregisters iscsi transport and the cxgb3 module 1422 */ 1423 static void __exit cxgb3i_exit_module(void) 1424 { 1425 cxgb3_unregister_client(&t3_client); 1426 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3); 1427 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1428 } 1429 1430 module_init(cxgb3i_init_module); 1431 module_exit(cxgb3i_exit_module); 1432