1 /* 2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management 3 * 4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved. 5 * 6 * This program is distributed in the hope that it will be useful, but WITHOUT 7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 9 * release for licensing terms and conditions. 10 * 11 * Written by: Dimitris Michailidis (dm@chelsio.com) 12 * Karen Xie (kxie@chelsio.com) 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 16 17 #include <linux/version.h> 18 #include <linux/module.h> 19 #include <linux/moduleparam.h> 20 #include <scsi/scsi_host.h> 21 22 #include "common.h" 23 #include "t3_cpl.h" 24 #include "t3cdev.h" 25 #include "cxgb3_defs.h" 26 #include "cxgb3_ctl_defs.h" 27 #include "cxgb3_offload.h" 28 #include "firmware_exports.h" 29 #include "cxgb3i.h" 30 31 static unsigned int dbg_level; 32 #include "../libcxgbi.h" 33 34 #define DRV_MODULE_NAME "cxgb3i" 35 #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver" 36 #define DRV_MODULE_VERSION "2.0.0" 37 #define DRV_MODULE_RELDATE "Jun. 2010" 38 39 static char version[] = 40 DRV_MODULE_DESC " " DRV_MODULE_NAME 41 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 42 43 MODULE_AUTHOR("Chelsio Communications, Inc."); 44 MODULE_DESCRIPTION(DRV_MODULE_DESC); 45 MODULE_VERSION(DRV_MODULE_VERSION); 46 MODULE_LICENSE("GPL"); 47 48 module_param(dbg_level, uint, 0644); 49 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)"); 50 51 static int cxgb3i_rcv_win = 256 * 1024; 52 module_param(cxgb3i_rcv_win, int, 0644); 53 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)"); 54 55 static int cxgb3i_snd_win = 128 * 1024; 56 module_param(cxgb3i_snd_win, int, 0644); 57 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)"); 58 59 static int cxgb3i_rx_credit_thres = 10 * 1024; 60 module_param(cxgb3i_rx_credit_thres, int, 0644); 61 MODULE_PARM_DESC(rx_credit_thres, 62 "RX credits return threshold in bytes (default=10KB)"); 63 64 static unsigned int cxgb3i_max_connect = 8 * 1024; 65 module_param(cxgb3i_max_connect, uint, 0644); 66 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)"); 67 68 static unsigned int cxgb3i_sport_base = 20000; 69 module_param(cxgb3i_sport_base, uint, 0644); 70 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)"); 71 72 static void cxgb3i_dev_open(struct t3cdev *); 73 static void cxgb3i_dev_close(struct t3cdev *); 74 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32); 75 76 static struct cxgb3_client t3_client = { 77 .name = DRV_MODULE_NAME, 78 .handlers = cxgb3i_cpl_handlers, 79 .add = cxgb3i_dev_open, 80 .remove = cxgb3i_dev_close, 81 .event_handler = cxgb3i_dev_event_handler, 82 }; 83 84 static struct scsi_host_template cxgb3i_host_template = { 85 .module = THIS_MODULE, 86 .name = DRV_MODULE_NAME, 87 .proc_name = DRV_MODULE_NAME, 88 .can_queue = CXGB3I_SCSI_HOST_QDEPTH, 89 .queuecommand = iscsi_queuecommand, 90 .change_queue_depth = iscsi_change_queue_depth, 91 .sg_tablesize = SG_ALL, 92 .max_sectors = 0xFFFF, 93 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 94 .eh_abort_handler = iscsi_eh_abort, 95 .eh_device_reset_handler = iscsi_eh_device_reset, 96 .eh_target_reset_handler = iscsi_eh_recover_target, 97 .target_alloc = iscsi_target_alloc, 98 .use_clustering = DISABLE_CLUSTERING, 99 .this_id = -1, 100 }; 101 102 static struct iscsi_transport cxgb3i_iscsi_transport = { 103 .owner = THIS_MODULE, 104 .name = DRV_MODULE_NAME, 105 /* owner and name should be set already */ 106 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 107 | CAP_DATADGST | CAP_DIGEST_OFFLOAD | 108 CAP_PADDING_OFFLOAD, 109 .param_mask = ISCSI_MAX_RECV_DLENGTH | ISCSI_MAX_XMIT_DLENGTH | 110 ISCSI_HDRDGST_EN | ISCSI_DATADGST_EN | 111 ISCSI_INITIAL_R2T_EN | ISCSI_MAX_R2T | 112 ISCSI_IMM_DATA_EN | ISCSI_FIRST_BURST | 113 ISCSI_MAX_BURST | ISCSI_PDU_INORDER_EN | 114 ISCSI_DATASEQ_INORDER_EN | ISCSI_ERL | 115 ISCSI_CONN_PORT | ISCSI_CONN_ADDRESS | 116 ISCSI_EXP_STATSN | ISCSI_PERSISTENT_PORT | 117 ISCSI_PERSISTENT_ADDRESS | 118 ISCSI_TARGET_NAME | ISCSI_TPGT | 119 ISCSI_USERNAME | ISCSI_PASSWORD | 120 ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN | 121 ISCSI_FAST_ABORT | ISCSI_ABORT_TMO | 122 ISCSI_LU_RESET_TMO | ISCSI_TGT_RESET_TMO | 123 ISCSI_PING_TMO | ISCSI_RECV_TMO | 124 ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME, 125 .host_param_mask = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS | 126 ISCSI_HOST_INITIATOR_NAME | 127 ISCSI_HOST_NETDEV_NAME, 128 .get_host_param = cxgbi_get_host_param, 129 .set_host_param = cxgbi_set_host_param, 130 /* session management */ 131 .create_session = cxgbi_create_session, 132 .destroy_session = cxgbi_destroy_session, 133 .get_session_param = iscsi_session_get_param, 134 /* connection management */ 135 .create_conn = cxgbi_create_conn, 136 .bind_conn = cxgbi_bind_conn, 137 .destroy_conn = iscsi_tcp_conn_teardown, 138 .start_conn = iscsi_conn_start, 139 .stop_conn = iscsi_conn_stop, 140 .get_conn_param = cxgbi_get_conn_param, 141 .set_param = cxgbi_set_conn_param, 142 .get_stats = cxgbi_get_conn_stats, 143 /* pdu xmit req from user space */ 144 .send_pdu = iscsi_conn_send_pdu, 145 /* task */ 146 .init_task = iscsi_tcp_task_init, 147 .xmit_task = iscsi_tcp_task_xmit, 148 .cleanup_task = cxgbi_cleanup_task, 149 /* pdu */ 150 .alloc_pdu = cxgbi_conn_alloc_pdu, 151 .init_pdu = cxgbi_conn_init_pdu, 152 .xmit_pdu = cxgbi_conn_xmit_pdu, 153 .parse_pdu_itt = cxgbi_parse_pdu_itt, 154 /* TCP connect/disconnect */ 155 .ep_connect = cxgbi_ep_connect, 156 .ep_poll = cxgbi_ep_poll, 157 .ep_disconnect = cxgbi_ep_disconnect, 158 /* Error recovery timeout call */ 159 .session_recovery_timedout = iscsi_session_recovery_timedout, 160 }; 161 162 static struct scsi_transport_template *cxgb3i_stt; 163 164 /* 165 * CPL (Chelsio Protocol Language) defines a message passing interface between 166 * the host driver and Chelsio asic. 167 * The section below implments CPLs that related to iscsi tcp connection 168 * open/close/abort and data send/receive. 169 */ 170 171 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion); 172 173 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 174 const struct l2t_entry *e) 175 { 176 unsigned int wscale = cxgbi_sock_compute_wscale(cxgb3i_rcv_win); 177 struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head; 178 179 skb->priority = CPL_PRIORITY_SETUP; 180 181 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 182 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid)); 183 req->local_port = csk->saddr.sin_port; 184 req->peer_port = csk->daddr.sin_port; 185 req->local_ip = csk->saddr.sin_addr.s_addr; 186 req->peer_ip = csk->daddr.sin_addr.s_addr; 187 188 req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS | 189 V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) | 190 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); 191 req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) | 192 V_RCV_BUFSIZ(cxgb3i_rcv_win>>10)); 193 194 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 195 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n", 196 csk, csk->state, csk->flags, csk->atid, 197 &req->local_ip, ntohs(req->local_port), 198 &req->peer_ip, ntohs(req->peer_port), 199 csk->mss_idx, e->idx, e->smt_idx); 200 201 l2t_send(csk->cdev->lldev, skb, csk->l2t); 202 } 203 204 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb) 205 { 206 cxgbi_sock_act_open_req_arp_failure(NULL, skb); 207 } 208 209 /* 210 * CPL connection close request: host -> 211 * 212 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to 213 * the write queue (i.e., after any unsent txt data). 214 */ 215 static void send_close_req(struct cxgbi_sock *csk) 216 { 217 struct sk_buff *skb = csk->cpl_close; 218 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 219 unsigned int tid = csk->tid; 220 221 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 222 "csk 0x%p,%u,0x%lx,%u.\n", 223 csk, csk->state, csk->flags, csk->tid); 224 225 csk->cpl_close = NULL; 226 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 227 req->wr.wr_lo = htonl(V_WR_TID(tid)); 228 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 229 req->rsvd = htonl(csk->write_seq); 230 231 cxgbi_sock_skb_entail(csk, skb); 232 if (csk->state >= CTP_ESTABLISHED) 233 push_tx_frames(csk, 1); 234 } 235 236 /* 237 * CPL connection abort request: host -> 238 * 239 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs 240 * for the same connection and also that we do not try to send a message 241 * after the connection has closed. 242 */ 243 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb) 244 { 245 struct cpl_abort_req *req = cplhdr(skb); 246 247 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 248 "t3dev 0x%p, tid %u, skb 0x%p.\n", 249 tdev, GET_TID(req), skb); 250 req->cmd = CPL_ABORT_NO_RST; 251 cxgb3_ofld_send(tdev, skb); 252 } 253 254 static void send_abort_req(struct cxgbi_sock *csk) 255 { 256 struct sk_buff *skb = csk->cpl_abort_req; 257 struct cpl_abort_req *req; 258 259 if (unlikely(csk->state == CTP_ABORTING || !skb)) 260 return; 261 cxgbi_sock_set_state(csk, CTP_ABORTING); 262 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 263 /* Purge the send queue so we don't send anything after an abort. */ 264 cxgbi_sock_purge_write_queue(csk); 265 266 csk->cpl_abort_req = NULL; 267 req = (struct cpl_abort_req *)skb->head; 268 skb->priority = CPL_PRIORITY_DATA; 269 set_arp_failure_handler(skb, abort_arp_failure); 270 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 271 req->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 272 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 273 req->rsvd0 = htonl(csk->snd_nxt); 274 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 275 req->cmd = CPL_ABORT_SEND_RST; 276 277 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 278 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 279 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 280 req->rsvd1); 281 282 l2t_send(csk->cdev->lldev, skb, csk->l2t); 283 } 284 285 /* 286 * CPL connection abort reply: host -> 287 * 288 * Send an ABORT_RPL message in response of the ABORT_REQ received. 289 */ 290 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 291 { 292 struct sk_buff *skb = csk->cpl_abort_rpl; 293 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 294 295 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 296 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 297 csk, csk->state, csk->flags, csk->tid, rst_status); 298 299 csk->cpl_abort_rpl = NULL; 300 skb->priority = CPL_PRIORITY_DATA; 301 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 302 rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 303 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 304 rpl->cmd = rst_status; 305 cxgb3_ofld_send(csk->cdev->lldev, skb); 306 } 307 308 /* 309 * CPL connection rx data ack: host -> 310 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 311 * credits sent. 312 */ 313 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 314 { 315 struct sk_buff *skb; 316 struct cpl_rx_data_ack *req; 317 u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 318 319 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 320 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n", 321 csk, csk->state, csk->flags, csk->tid, credits, dack); 322 323 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 324 if (!skb) { 325 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 326 return 0; 327 } 328 req = (struct cpl_rx_data_ack *)skb->head; 329 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 330 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid)); 331 req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) | 332 V_RX_CREDITS(credits)); 333 skb->priority = CPL_PRIORITY_ACK; 334 cxgb3_ofld_send(csk->cdev->lldev, skb); 335 return credits; 336 } 337 338 /* 339 * CPL connection tx data: host -> 340 * 341 * Send iscsi PDU via TX_DATA CPL message. Returns the number of 342 * credits sent. 343 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of 344 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3). 345 */ 346 347 static unsigned int wrlen __read_mostly; 348 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly; 349 350 static void init_wr_tab(unsigned int wr_len) 351 { 352 int i; 353 354 if (skb_wrs[1]) /* already initialized */ 355 return; 356 for (i = 1; i < SKB_WR_LIST_SIZE; i++) { 357 int sgl_len = (3 * i) / 2 + (i & 1); 358 359 sgl_len += 3; 360 skb_wrs[i] = (sgl_len <= wr_len 361 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1)); 362 } 363 wrlen = wr_len * 8; 364 } 365 366 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 367 int len, int req_completion) 368 { 369 struct tx_data_wr *req; 370 struct l2t_entry *l2t = csk->l2t; 371 372 skb_reset_transport_header(skb); 373 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req)); 374 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) | 375 (req_completion ? F_WR_COMPL : 0)); 376 req->wr_lo = htonl(V_WR_TID(csk->tid)); 377 /* len includes the length of any HW ULP additions */ 378 req->len = htonl(len); 379 /* V_TX_ULP_SUBMODE sets both the mode and submode */ 380 req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) | 381 V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1))); 382 req->sndseq = htonl(csk->snd_nxt); 383 req->param = htonl(V_TX_PORT(l2t->smt_idx)); 384 385 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 386 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | 387 V_TX_CPU_IDX(csk->rss_qid)); 388 /* sendbuffer is in units of 32KB. */ 389 req->param |= htonl(V_TX_SNDBUF(cxgb3i_snd_win >> 15)); 390 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 391 } 392 } 393 394 /** 395 * push_tx_frames -- start transmit 396 * @c3cn: the offloaded connection 397 * @req_completion: request wr_ack or not 398 * 399 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a 400 * connection's send queue and sends them on to T3. Must be called with the 401 * connection's lock held. Returns the amount of send buffer space that was 402 * freed as a result of sending queued data to T3. 403 */ 404 405 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb) 406 { 407 kfree_skb(skb); 408 } 409 410 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 411 { 412 int total_size = 0; 413 struct sk_buff *skb; 414 415 if (unlikely(csk->state < CTP_ESTABLISHED || 416 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 417 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 418 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 419 csk, csk->state, csk->flags, csk->tid); 420 return 0; 421 } 422 423 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { 424 int len = skb->len; /* length before skb_push */ 425 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len); 426 int wrs_needed = skb_wrs[frags]; 427 428 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen) 429 wrs_needed = 1; 430 431 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1); 432 433 if (csk->wr_cred < wrs_needed) { 434 log_debug(1 << CXGBI_DBG_PDU_TX, 435 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n", 436 csk, skb->len, skb->data_len, frags, 437 wrs_needed, csk->wr_cred); 438 break; 439 } 440 441 __skb_unlink(skb, &csk->write_queue); 442 skb->priority = CPL_PRIORITY_DATA; 443 skb->csum = wrs_needed; /* remember this until the WR_ACK */ 444 csk->wr_cred -= wrs_needed; 445 csk->wr_una_cred += wrs_needed; 446 cxgbi_sock_enqueue_wr(csk, skb); 447 448 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 449 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, " 450 "left %u, unack %u.\n", 451 csk, skb->len, skb->data_len, frags, skb->csum, 452 csk->wr_cred, csk->wr_una_cred); 453 454 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 455 if ((req_completion && 456 csk->wr_una_cred == wrs_needed) || 457 csk->wr_una_cred >= csk->wr_max_cred / 2) { 458 req_completion = 1; 459 csk->wr_una_cred = 0; 460 } 461 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 462 make_tx_data_wr(csk, skb, len, req_completion); 463 csk->snd_nxt += len; 464 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 465 } 466 total_size += skb->truesize; 467 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 468 "csk 0x%p, tid 0x%x, send skb 0x%p.\n", 469 csk, csk->tid, skb); 470 set_arp_failure_handler(skb, arp_failure_skb_discard); 471 l2t_send(csk->cdev->lldev, skb, csk->l2t); 472 } 473 return total_size; 474 } 475 476 /* 477 * Process a CPL_ACT_ESTABLISH message: -> host 478 * Updates connection state from an active establish CPL message. Runs with 479 * the connection lock held. 480 */ 481 482 static inline void free_atid(struct cxgbi_sock *csk) 483 { 484 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 485 cxgb3_free_atid(csk->cdev->lldev, csk->atid); 486 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 487 cxgbi_sock_put(csk); 488 } 489 } 490 491 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 492 { 493 struct cxgbi_sock *csk = ctx; 494 struct cpl_act_establish *req = cplhdr(skb); 495 unsigned int tid = GET_TID(req); 496 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 497 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ 498 499 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 500 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 501 atid, atid, csk, csk->state, csk->flags, rcv_isn); 502 503 cxgbi_sock_get(csk); 504 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 505 csk->tid = tid; 506 cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid); 507 508 free_atid(csk); 509 510 csk->rss_qid = G_QNUM(ntohs(skb->csum)); 511 512 spin_lock_bh(&csk->lock); 513 if (csk->retry_timer.function) { 514 del_timer(&csk->retry_timer); 515 csk->retry_timer.function = NULL; 516 } 517 518 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 519 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 520 csk, csk->state, csk->flags, csk->tid); 521 522 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 523 if (cxgb3i_rcv_win > (M_RCV_BUFSIZ << 10)) 524 csk->rcv_wup -= cxgb3i_rcv_win - (M_RCV_BUFSIZ << 10); 525 526 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 527 528 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 529 /* upper layer has requested closing */ 530 send_abort_req(csk); 531 else { 532 if (skb_queue_len(&csk->write_queue)) 533 push_tx_frames(csk, 1); 534 cxgbi_conn_tx_open(csk); 535 } 536 537 spin_unlock_bh(&csk->lock); 538 __kfree_skb(skb); 539 return 0; 540 } 541 542 /* 543 * Process a CPL_ACT_OPEN_RPL message: -> host 544 * Handle active open failures. 545 */ 546 static int act_open_rpl_status_to_errno(int status) 547 { 548 switch (status) { 549 case CPL_ERR_CONN_RESET: 550 return -ECONNREFUSED; 551 case CPL_ERR_ARP_MISS: 552 return -EHOSTUNREACH; 553 case CPL_ERR_CONN_TIMEDOUT: 554 return -ETIMEDOUT; 555 case CPL_ERR_TCAM_FULL: 556 return -ENOMEM; 557 case CPL_ERR_CONN_EXIST: 558 return -EADDRINUSE; 559 default: 560 return -EIO; 561 } 562 } 563 564 static void act_open_retry_timer(unsigned long data) 565 { 566 struct sk_buff *skb; 567 struct cxgbi_sock *csk = (struct cxgbi_sock *)data; 568 569 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 570 "csk 0x%p,%u,0x%lx,%u.\n", 571 csk, csk->state, csk->flags, csk->tid); 572 573 cxgbi_sock_get(csk); 574 spin_lock_bh(&csk->lock); 575 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC); 576 if (!skb) 577 cxgbi_sock_fail_act_open(csk, -ENOMEM); 578 else { 579 skb->sk = (struct sock *)csk; 580 set_arp_failure_handler(skb, act_open_arp_failure); 581 send_act_open_req(csk, skb, csk->l2t); 582 } 583 spin_unlock_bh(&csk->lock); 584 cxgbi_sock_put(csk); 585 } 586 587 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 588 { 589 struct cxgbi_sock *csk = ctx; 590 struct cpl_act_open_rpl *rpl = cplhdr(skb); 591 592 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n", 593 csk, csk->state, csk->flags, csk->atid, rpl->status, 594 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 595 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 596 597 if (rpl->status != CPL_ERR_TCAM_FULL && 598 rpl->status != CPL_ERR_CONN_EXIST && 599 rpl->status != CPL_ERR_ARP_MISS) 600 cxgb3_queue_tid_release(tdev, GET_TID(rpl)); 601 602 cxgbi_sock_get(csk); 603 spin_lock_bh(&csk->lock); 604 if (rpl->status == CPL_ERR_CONN_EXIST && 605 csk->retry_timer.function != act_open_retry_timer) { 606 csk->retry_timer.function = act_open_retry_timer; 607 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 608 } else 609 cxgbi_sock_fail_act_open(csk, 610 act_open_rpl_status_to_errno(rpl->status)); 611 612 spin_unlock_bh(&csk->lock); 613 cxgbi_sock_put(csk); 614 __kfree_skb(skb); 615 return 0; 616 } 617 618 /* 619 * Process PEER_CLOSE CPL messages: -> host 620 * Handle peer FIN. 621 */ 622 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 623 { 624 struct cxgbi_sock *csk = ctx; 625 626 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 627 "csk 0x%p,%u,0x%lx,%u.\n", 628 csk, csk->state, csk->flags, csk->tid); 629 630 cxgbi_sock_rcv_peer_close(csk); 631 __kfree_skb(skb); 632 return 0; 633 } 634 635 /* 636 * Process CLOSE_CONN_RPL CPL message: -> host 637 * Process a peer ACK to our FIN. 638 */ 639 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb, 640 void *ctx) 641 { 642 struct cxgbi_sock *csk = ctx; 643 struct cpl_close_con_rpl *rpl = cplhdr(skb); 644 645 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 646 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n", 647 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt)); 648 649 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 650 __kfree_skb(skb); 651 return 0; 652 } 653 654 /* 655 * Process ABORT_REQ_RSS CPL message: -> host 656 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this 657 * request except that we need to reply to it. 658 */ 659 660 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 661 int *need_rst) 662 { 663 switch (abort_reason) { 664 case CPL_ERR_BAD_SYN: /* fall through */ 665 case CPL_ERR_CONN_RESET: 666 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET; 667 case CPL_ERR_XMIT_TIMEDOUT: 668 case CPL_ERR_PERSIST_TIMEDOUT: 669 case CPL_ERR_FINWAIT2_TIMEDOUT: 670 case CPL_ERR_KEEPALIVE_TIMEDOUT: 671 return -ETIMEDOUT; 672 default: 673 return -EIO; 674 } 675 } 676 677 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 678 { 679 const struct cpl_abort_req_rss *req = cplhdr(skb); 680 struct cxgbi_sock *csk = ctx; 681 int rst_status = CPL_ABORT_NO_RST; 682 683 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 684 "csk 0x%p,%u,0x%lx,%u.\n", 685 csk, csk->state, csk->flags, csk->tid); 686 687 if (req->status == CPL_ERR_RTX_NEG_ADVICE || 688 req->status == CPL_ERR_PERSIST_NEG_ADVICE) { 689 goto done; 690 } 691 692 cxgbi_sock_get(csk); 693 spin_lock_bh(&csk->lock); 694 695 if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) { 696 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 697 cxgbi_sock_set_state(csk, CTP_ABORTING); 698 goto out; 699 } 700 701 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 702 send_abort_rpl(csk, rst_status); 703 704 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 705 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 706 cxgbi_sock_closed(csk); 707 } 708 709 out: 710 spin_unlock_bh(&csk->lock); 711 cxgbi_sock_put(csk); 712 done: 713 __kfree_skb(skb); 714 return 0; 715 } 716 717 /* 718 * Process ABORT_RPL_RSS CPL message: -> host 719 * Process abort replies. We only process these messages if we anticipate 720 * them as the coordination between SW and HW in this area is somewhat lacking 721 * and sometimes we get ABORT_RPLs after we are done with the connection that 722 * originated the ABORT_REQ. 723 */ 724 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 725 { 726 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 727 struct cxgbi_sock *csk = ctx; 728 729 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 730 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n", 731 rpl->status, csk, csk ? csk->state : 0, 732 csk ? csk->flags : 0UL); 733 /* 734 * Ignore replies to post-close aborts indicating that the abort was 735 * requested too late. These connections are terminated when we get 736 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss 737 * arrives the TID is either no longer used or it has been recycled. 738 */ 739 if (rpl->status == CPL_ERR_ABORT_FAILED) 740 goto rel_skb; 741 /* 742 * Sometimes we've already closed the connection, e.g., a post-close 743 * abort races with ABORT_REQ_RSS, the latter frees the connection 744 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, 745 * but FW turns the ABORT_REQ into a regular one and so we get 746 * ABORT_RPL_RSS with status 0 and no connection. 747 */ 748 if (csk) 749 cxgbi_sock_rcv_abort_rpl(csk); 750 rel_skb: 751 __kfree_skb(skb); 752 return 0; 753 } 754 755 /* 756 * Process RX_ISCSI_HDR CPL message: -> host 757 * Handle received PDUs, the payload could be DDP'ed. If not, the payload 758 * follow after the bhs. 759 */ 760 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx) 761 { 762 struct cxgbi_sock *csk = ctx; 763 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb); 764 struct cpl_iscsi_hdr_norss data_cpl; 765 struct cpl_rx_data_ddp_norss ddp_cpl; 766 unsigned int hdr_len, data_len, status; 767 unsigned int len; 768 int err; 769 770 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 771 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n", 772 csk, csk->state, csk->flags, csk->tid, skb, skb->len); 773 774 spin_lock_bh(&csk->lock); 775 776 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 777 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 778 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 779 csk, csk->state, csk->flags, csk->tid); 780 if (csk->state != CTP_ABORTING) 781 goto abort_conn; 782 else 783 goto discard; 784 } 785 786 cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq); 787 cxgbi_skcb_flags(skb) = 0; 788 789 skb_reset_transport_header(skb); 790 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr)); 791 792 len = hdr_len = ntohs(hdr_cpl->len); 793 /* msg coalesce is off or not enough data received */ 794 if (skb->len <= hdr_len) { 795 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n", 796 csk->cdev->ports[csk->port_id]->name, csk->tid, 797 skb->len, hdr_len); 798 goto abort_conn; 799 } 800 cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED); 801 802 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl, 803 sizeof(ddp_cpl)); 804 if (err < 0) { 805 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n", 806 csk->cdev->ports[csk->port_id]->name, csk->tid, 807 skb->len, sizeof(ddp_cpl), err); 808 goto abort_conn; 809 } 810 811 cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); 812 cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len); 813 cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc); 814 status = ntohl(ddp_cpl.ddp_status); 815 816 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 817 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n", 818 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status); 819 820 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) 821 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); 822 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) 823 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); 824 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) 825 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); 826 827 if (skb->len > (hdr_len + sizeof(ddp_cpl))) { 828 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl)); 829 if (err < 0) { 830 pr_err("%s: tid %u, cp %zu/%u failed %d.\n", 831 csk->cdev->ports[csk->port_id]->name, 832 csk->tid, sizeof(data_cpl), skb->len, err); 833 goto abort_conn; 834 } 835 data_len = ntohs(data_cpl.len); 836 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX, 837 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n", 838 skb, data_len, cxgbi_skcb_rx_pdulen(skb), status); 839 len += sizeof(data_cpl) + data_len; 840 } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) 841 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); 842 843 csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb); 844 __pskb_trim(skb, len); 845 __skb_queue_tail(&csk->receive_queue, skb); 846 cxgbi_conn_pdu_ready(csk); 847 848 spin_unlock_bh(&csk->lock); 849 return 0; 850 851 abort_conn: 852 send_abort_req(csk); 853 discard: 854 spin_unlock_bh(&csk->lock); 855 __kfree_skb(skb); 856 return 0; 857 } 858 859 /* 860 * Process TX_DATA_ACK CPL messages: -> host 861 * Process an acknowledgment of WR completion. Advance snd_una and send the 862 * next batch of work requests from the write queue. 863 */ 864 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 865 { 866 struct cxgbi_sock *csk = ctx; 867 struct cpl_wr_ack *hdr = cplhdr(skb); 868 869 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 870 "csk 0x%p,%u,0x%lx,%u, cr %u.\n", 871 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits)); 872 873 cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1); 874 __kfree_skb(skb); 875 return 0; 876 } 877 878 /* 879 * for each connection, pre-allocate skbs needed for close/abort requests. So 880 * that we can service the request right away. 881 */ 882 static int alloc_cpls(struct cxgbi_sock *csk) 883 { 884 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0, 885 GFP_KERNEL); 886 if (!csk->cpl_close) 887 return -ENOMEM; 888 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0, 889 GFP_KERNEL); 890 if (!csk->cpl_abort_req) 891 goto free_cpl_skbs; 892 893 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0, 894 GFP_KERNEL); 895 if (!csk->cpl_abort_rpl) 896 goto free_cpl_skbs; 897 898 return 0; 899 900 free_cpl_skbs: 901 cxgbi_sock_free_cpl_skbs(csk); 902 return -ENOMEM; 903 } 904 905 /** 906 * release_offload_resources - release offload resource 907 * @c3cn: the offloaded iscsi tcp connection. 908 * Release resources held by an offload connection (TID, L2T entry, etc.) 909 */ 910 static void l2t_put(struct cxgbi_sock *csk) 911 { 912 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 913 914 if (csk->l2t) { 915 l2t_release(L2DATA(t3dev), csk->l2t); 916 csk->l2t = NULL; 917 cxgbi_sock_put(csk); 918 } 919 } 920 921 static void release_offload_resources(struct cxgbi_sock *csk) 922 { 923 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 924 925 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 926 "csk 0x%p,%u,0x%lx,%u.\n", 927 csk, csk->state, csk->flags, csk->tid); 928 929 csk->rss_qid = 0; 930 cxgbi_sock_free_cpl_skbs(csk); 931 932 if (csk->wr_cred != csk->wr_max_cred) { 933 cxgbi_sock_purge_wr_queue(csk); 934 cxgbi_sock_reset_wr_list(csk); 935 } 936 l2t_put(csk); 937 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 938 free_atid(csk); 939 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 940 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid); 941 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 942 cxgbi_sock_put(csk); 943 } 944 csk->dst = NULL; 945 csk->cdev = NULL; 946 } 947 948 static void update_address(struct cxgbi_hba *chba) 949 { 950 if (chba->ipv4addr) { 951 if (chba->vdev && 952 chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) { 953 cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr); 954 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 955 pr_info("%s set %pI4.\n", 956 chba->vdev->name, &chba->ipv4addr); 957 } else if (chba->ipv4addr != 958 cxgb3i_get_private_ipv4addr(chba->ndev)) { 959 cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr); 960 pr_info("%s set %pI4.\n", 961 chba->ndev->name, &chba->ipv4addr); 962 } 963 } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) { 964 if (chba->vdev) 965 cxgb3i_set_private_ipv4addr(chba->vdev, 0); 966 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 967 } 968 } 969 970 static int init_act_open(struct cxgbi_sock *csk) 971 { 972 struct dst_entry *dst = csk->dst; 973 struct cxgbi_device *cdev = csk->cdev; 974 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 975 struct net_device *ndev = cdev->ports[csk->port_id]; 976 struct cxgbi_hba *chba = cdev->hbas[csk->port_id]; 977 struct sk_buff *skb = NULL; 978 979 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 980 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags); 981 982 update_address(chba); 983 if (chba->ipv4addr) 984 csk->saddr.sin_addr.s_addr = chba->ipv4addr; 985 986 csk->rss_qid = 0; 987 csk->l2t = t3_l2t_get(t3dev, dst->neighbour, ndev); 988 if (!csk->l2t) { 989 pr_err("NO l2t available.\n"); 990 return -EINVAL; 991 } 992 cxgbi_sock_get(csk); 993 994 csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk); 995 if (csk->atid < 0) { 996 pr_err("NO atid available.\n"); 997 goto rel_resource; 998 } 999 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 1000 cxgbi_sock_get(csk); 1001 1002 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL); 1003 if (!skb) 1004 goto rel_resource; 1005 skb->sk = (struct sock *)csk; 1006 set_arp_failure_handler(skb, act_open_arp_failure); 1007 1008 csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1; 1009 csk->wr_una_cred = 0; 1010 csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst)); 1011 cxgbi_sock_reset_wr_list(csk); 1012 csk->err = 0; 1013 1014 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1015 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n", 1016 csk, csk->state, csk->flags, 1017 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 1018 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 1019 1020 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1021 send_act_open_req(csk, skb, csk->l2t); 1022 return 0; 1023 1024 rel_resource: 1025 if (skb) 1026 __kfree_skb(skb); 1027 return -EINVAL; 1028 } 1029 1030 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = { 1031 [CPL_ACT_ESTABLISH] = do_act_establish, 1032 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1033 [CPL_PEER_CLOSE] = do_peer_close, 1034 [CPL_ABORT_REQ_RSS] = do_abort_req, 1035 [CPL_ABORT_RPL_RSS] = do_abort_rpl, 1036 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1037 [CPL_TX_DMA_ACK] = do_wr_ack, 1038 [CPL_ISCSI_HDR] = do_iscsi_hdr, 1039 }; 1040 1041 /** 1042 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found 1043 * @cdev: cxgbi adapter 1044 */ 1045 int cxgb3i_ofld_init(struct cxgbi_device *cdev) 1046 { 1047 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 1048 struct adap_ports port; 1049 struct ofld_page_info rx_page_info; 1050 unsigned int wr_len; 1051 int rc; 1052 1053 if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 || 1054 t3dev->ctl(t3dev, GET_PORTS, &port) < 0 || 1055 t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) { 1056 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev); 1057 return -EINVAL; 1058 } 1059 1060 if (cxgb3i_max_connect > CXGBI_MAX_CONN) 1061 cxgb3i_max_connect = CXGBI_MAX_CONN; 1062 1063 rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base, 1064 cxgb3i_max_connect); 1065 if (rc < 0) 1066 return rc; 1067 1068 init_wr_tab(wr_len); 1069 cdev->csk_release_offload_resources = release_offload_resources; 1070 cdev->csk_push_tx_frames = push_tx_frames; 1071 cdev->csk_send_abort_req = send_abort_req; 1072 cdev->csk_send_close_req = send_close_req; 1073 cdev->csk_send_rx_credits = send_rx_credits; 1074 cdev->csk_alloc_cpls = alloc_cpls; 1075 cdev->csk_init_act_open = init_act_open; 1076 1077 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1078 return 0; 1079 } 1080 1081 /* 1082 * functions to program the pagepod in h/w 1083 */ 1084 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr) 1085 { 1086 struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head; 1087 1088 memset(req, 0, sizeof(*req)); 1089 1090 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 1091 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) | 1092 V_ULPTX_CMD(ULP_MEM_WRITE)); 1093 req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE >> 5) | 1094 V_ULPTX_NFLITS((PPOD_SIZE >> 3) + 1)); 1095 } 1096 1097 static int ddp_set_map(struct cxgbi_sock *csk, struct cxgbi_pagepod_hdr *hdr, 1098 unsigned int idx, unsigned int npods, 1099 struct cxgbi_gather_list *gl) 1100 { 1101 struct cxgbi_device *cdev = csk->cdev; 1102 struct cxgbi_ddp_info *ddp = cdev->ddp; 1103 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit; 1104 int i; 1105 1106 log_debug(1 << CXGBI_DBG_DDP, 1107 "csk 0x%p, idx %u, npods %u, gl 0x%p.\n", 1108 csk, idx, npods, gl); 1109 1110 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) { 1111 struct sk_buff *skb = ddp->gl_skb[idx]; 1112 1113 /* hold on to the skb until we clear the ddp mapping */ 1114 skb_get(skb); 1115 1116 ulp_mem_io_set_hdr(skb, pm_addr); 1117 cxgbi_ddp_ppod_set((struct cxgbi_pagepod *)(skb->head + 1118 sizeof(struct ulp_mem_io)), 1119 hdr, gl, i * PPOD_PAGES_MAX); 1120 skb->priority = CPL_PRIORITY_CONTROL; 1121 cxgb3_ofld_send(cdev->lldev, skb); 1122 } 1123 return 0; 1124 } 1125 1126 static void ddp_clear_map(struct cxgbi_hba *chba, unsigned int tag, 1127 unsigned int idx, unsigned int npods) 1128 { 1129 struct cxgbi_device *cdev = chba->cdev; 1130 struct cxgbi_ddp_info *ddp = cdev->ddp; 1131 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ddp->llimit; 1132 int i; 1133 1134 log_debug(1 << CXGBI_DBG_DDP, 1135 "cdev 0x%p, idx %u, npods %u, tag 0x%x.\n", 1136 cdev, idx, npods, tag); 1137 1138 for (i = 0; i < npods; i++, idx++, pm_addr += PPOD_SIZE) { 1139 struct sk_buff *skb = ddp->gl_skb[idx]; 1140 1141 if (!skb) { 1142 pr_err("tag 0x%x, 0x%x, %d/%u, skb NULL.\n", 1143 tag, idx, i, npods); 1144 continue; 1145 } 1146 ddp->gl_skb[idx] = NULL; 1147 memset(skb->head + sizeof(struct ulp_mem_io), 0, PPOD_SIZE); 1148 ulp_mem_io_set_hdr(skb, pm_addr); 1149 skb->priority = CPL_PRIORITY_CONTROL; 1150 cxgb3_ofld_send(cdev->lldev, skb); 1151 } 1152 } 1153 1154 static void ddp_free_gl_skb(struct cxgbi_ddp_info *ddp, int idx, int cnt) 1155 { 1156 int i; 1157 1158 log_debug(1 << CXGBI_DBG_DDP, 1159 "ddp 0x%p, idx %d, cnt %d.\n", ddp, idx, cnt); 1160 1161 for (i = 0; i < cnt; i++, idx++) 1162 if (ddp->gl_skb[idx]) { 1163 kfree_skb(ddp->gl_skb[idx]); 1164 ddp->gl_skb[idx] = NULL; 1165 } 1166 } 1167 1168 static int ddp_alloc_gl_skb(struct cxgbi_ddp_info *ddp, int idx, 1169 int cnt, gfp_t gfp) 1170 { 1171 int i; 1172 1173 log_debug(1 << CXGBI_DBG_DDP, 1174 "ddp 0x%p, idx %d, cnt %d.\n", ddp, idx, cnt); 1175 1176 for (i = 0; i < cnt; i++) { 1177 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + 1178 PPOD_SIZE, 0, gfp); 1179 if (skb) 1180 ddp->gl_skb[idx + i] = skb; 1181 else { 1182 ddp_free_gl_skb(ddp, idx, i); 1183 return -ENOMEM; 1184 } 1185 } 1186 return 0; 1187 } 1188 1189 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, 1190 unsigned int tid, int pg_idx, bool reply) 1191 { 1192 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1193 GFP_KERNEL); 1194 struct cpl_set_tcb_field *req; 1195 u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0; 1196 1197 log_debug(1 << CXGBI_DBG_DDP, 1198 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx); 1199 if (!skb) 1200 return -ENOMEM; 1201 1202 /* set up ulp submode and page size */ 1203 req = (struct cpl_set_tcb_field *)skb->head; 1204 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1205 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1206 req->reply = V_NO_REPLY(reply ? 0 : 1); 1207 req->cpu_idx = 0; 1208 req->word = htons(31); 1209 req->mask = cpu_to_be64(0xF0000000); 1210 req->val = cpu_to_be64(val << 28); 1211 skb->priority = CPL_PRIORITY_CONTROL; 1212 1213 cxgb3_ofld_send(csk->cdev->lldev, skb); 1214 return 0; 1215 } 1216 1217 /** 1218 * cxgb3i_setup_conn_digest - setup conn. digest setting 1219 * @csk: cxgb tcp socket 1220 * @tid: connection id 1221 * @hcrc: header digest enabled 1222 * @dcrc: data digest enabled 1223 * @reply: request reply from h/w 1224 * set up the iscsi digest settings for a connection identified by tid 1225 */ 1226 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 1227 int hcrc, int dcrc, int reply) 1228 { 1229 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1230 GFP_KERNEL); 1231 struct cpl_set_tcb_field *req; 1232 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0); 1233 1234 log_debug(1 << CXGBI_DBG_DDP, 1235 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc); 1236 if (!skb) 1237 return -ENOMEM; 1238 1239 /* set up ulp submode and page size */ 1240 req = (struct cpl_set_tcb_field *)skb->head; 1241 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1242 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1243 req->reply = V_NO_REPLY(reply ? 0 : 1); 1244 req->cpu_idx = 0; 1245 req->word = htons(31); 1246 req->mask = cpu_to_be64(0x0F000000); 1247 req->val = cpu_to_be64(val << 24); 1248 skb->priority = CPL_PRIORITY_CONTROL; 1249 1250 cxgb3_ofld_send(csk->cdev->lldev, skb); 1251 return 0; 1252 } 1253 1254 /** 1255 * t3_ddp_cleanup - release the cxgb3 adapter's ddp resource 1256 * @cdev: cxgb3i adapter 1257 * release all the resource held by the ddp pagepod manager for a given 1258 * adapter if needed 1259 */ 1260 1261 static void t3_ddp_cleanup(struct cxgbi_device *cdev) 1262 { 1263 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; 1264 1265 if (cxgbi_ddp_cleanup(cdev)) { 1266 pr_info("t3dev 0x%p, ulp_iscsi no more user.\n", tdev); 1267 tdev->ulp_iscsi = NULL; 1268 } 1269 } 1270 1271 /** 1272 * ddp_init - initialize the cxgb3 adapter's ddp resource 1273 * @cdev: cxgb3i adapter 1274 * initialize the ddp pagepod manager for a given adapter 1275 */ 1276 static int cxgb3i_ddp_init(struct cxgbi_device *cdev) 1277 { 1278 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; 1279 struct cxgbi_ddp_info *ddp = tdev->ulp_iscsi; 1280 struct ulp_iscsi_info uinfo; 1281 unsigned int pgsz_factor[4]; 1282 int err; 1283 1284 if (ddp) { 1285 kref_get(&ddp->refcnt); 1286 pr_warn("t3dev 0x%p, ddp 0x%p already set up.\n", 1287 tdev, tdev->ulp_iscsi); 1288 cdev->ddp = ddp; 1289 return -EALREADY; 1290 } 1291 1292 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo); 1293 if (err < 0) { 1294 pr_err("%s, failed to get iscsi param err=%d.\n", 1295 tdev->name, err); 1296 return err; 1297 } 1298 1299 err = cxgbi_ddp_init(cdev, uinfo.llimit, uinfo.ulimit, 1300 uinfo.max_txsz, uinfo.max_rxsz); 1301 if (err < 0) 1302 return err; 1303 1304 ddp = cdev->ddp; 1305 1306 uinfo.tagmask = ddp->idx_mask << PPOD_IDX_SHIFT; 1307 cxgbi_ddp_page_size_factor(pgsz_factor); 1308 uinfo.ulimit = uinfo.llimit + (ddp->nppods << PPOD_SIZE_SHIFT); 1309 1310 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo); 1311 if (err < 0) { 1312 pr_warn("%s unable to set iscsi param err=%d, ddp disabled.\n", 1313 tdev->name, err); 1314 cxgbi_ddp_cleanup(cdev); 1315 return err; 1316 } 1317 tdev->ulp_iscsi = ddp; 1318 1319 cdev->csk_ddp_free_gl_skb = ddp_free_gl_skb; 1320 cdev->csk_ddp_alloc_gl_skb = ddp_alloc_gl_skb; 1321 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 1322 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 1323 cdev->csk_ddp_set = ddp_set_map; 1324 cdev->csk_ddp_clear = ddp_clear_map; 1325 1326 pr_info("tdev 0x%p, nppods %u, bits %u, mask 0x%x,0x%x pkt %u/%u, " 1327 "%u/%u.\n", 1328 tdev, ddp->nppods, ddp->idx_bits, ddp->idx_mask, 1329 ddp->rsvd_tag_mask, ddp->max_txsz, uinfo.max_txsz, 1330 ddp->max_rxsz, uinfo.max_rxsz); 1331 return 0; 1332 } 1333 1334 static void cxgb3i_dev_close(struct t3cdev *t3dev) 1335 { 1336 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1337 1338 if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) { 1339 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0); 1340 return; 1341 } 1342 1343 cxgbi_device_unregister(cdev); 1344 } 1345 1346 /** 1347 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings 1348 * @t3dev: t3cdev adapter 1349 */ 1350 static void cxgb3i_dev_open(struct t3cdev *t3dev) 1351 { 1352 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1353 struct adapter *adapter = tdev2adap(t3dev); 1354 int i, err; 1355 1356 if (cdev) { 1357 pr_info("0x%p, updating.\n", cdev); 1358 return; 1359 } 1360 1361 cdev = cxgbi_device_register(0, adapter->params.nports); 1362 if (!cdev) { 1363 pr_warn("device 0x%p register failed.\n", t3dev); 1364 return; 1365 } 1366 1367 cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET; 1368 cdev->lldev = t3dev; 1369 cdev->pdev = adapter->pdev; 1370 cdev->ports = adapter->port; 1371 cdev->nports = adapter->params.nports; 1372 cdev->mtus = adapter->params.mtus; 1373 cdev->nmtus = NMTUS; 1374 cdev->snd_win = cxgb3i_snd_win; 1375 cdev->rcv_win = cxgb3i_rcv_win; 1376 cdev->rx_credit_thres = cxgb3i_rx_credit_thres; 1377 cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN; 1378 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss); 1379 cdev->dev_ddp_cleanup = t3_ddp_cleanup; 1380 cdev->itp = &cxgb3i_iscsi_transport; 1381 1382 err = cxgb3i_ddp_init(cdev); 1383 if (err) { 1384 pr_info("0x%p ddp init failed\n", cdev); 1385 goto err_out; 1386 } 1387 1388 err = cxgb3i_ofld_init(cdev); 1389 if (err) { 1390 pr_info("0x%p offload init failed\n", cdev); 1391 goto err_out; 1392 } 1393 1394 err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN, 1395 &cxgb3i_host_template, cxgb3i_stt); 1396 if (err) 1397 goto err_out; 1398 1399 for (i = 0; i < cdev->nports; i++) 1400 cdev->hbas[i]->ipv4addr = 1401 cxgb3i_get_private_ipv4addr(cdev->ports[i]); 1402 1403 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n", 1404 cdev, cdev ? cdev->flags : 0, t3dev, err); 1405 return; 1406 1407 err_out: 1408 cxgbi_device_unregister(cdev); 1409 } 1410 1411 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port) 1412 { 1413 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1414 1415 log_debug(1 << CXGBI_DBG_TOE, 1416 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n", 1417 t3dev, cdev, event, port); 1418 if (!cdev) 1419 return; 1420 1421 switch (event) { 1422 case OFFLOAD_STATUS_DOWN: 1423 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET; 1424 break; 1425 case OFFLOAD_STATUS_UP: 1426 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET; 1427 break; 1428 } 1429 } 1430 1431 /** 1432 * cxgb3i_init_module - module init entry point 1433 * 1434 * initialize any driver wide global data structures and register itself 1435 * with the cxgb3 module 1436 */ 1437 static int __init cxgb3i_init_module(void) 1438 { 1439 int rc; 1440 1441 printk(KERN_INFO "%s", version); 1442 1443 rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1444 if (rc < 0) 1445 return rc; 1446 1447 cxgb3_register_client(&t3_client); 1448 return 0; 1449 } 1450 1451 /** 1452 * cxgb3i_exit_module - module cleanup/exit entry point 1453 * 1454 * go through the driver hba list and for each hba, release any resource held. 1455 * and unregisters iscsi transport and the cxgb3 module 1456 */ 1457 static void __exit cxgb3i_exit_module(void) 1458 { 1459 cxgb3_unregister_client(&t3_client); 1460 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3); 1461 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1462 } 1463 1464 module_init(cxgb3i_init_module); 1465 module_exit(cxgb3i_exit_module); 1466