1 /* 2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management 3 * 4 * Copyright (C) 2003-2015 Chelsio Communications. All rights reserved. 5 * 6 * This program is distributed in the hope that it will be useful, but WITHOUT 7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 9 * release for licensing terms and conditions. 10 * 11 * Written by: Dimitris Michailidis (dm@chelsio.com) 12 * Karen Xie (kxie@chelsio.com) 13 */ 14 15 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 16 17 #include <linux/module.h> 18 #include <linux/moduleparam.h> 19 #include <scsi/scsi_host.h> 20 21 #include "common.h" 22 #include "t3_cpl.h" 23 #include "t3cdev.h" 24 #include "cxgb3_defs.h" 25 #include "cxgb3_ctl_defs.h" 26 #include "cxgb3_offload.h" 27 #include "firmware_exports.h" 28 #include "cxgb3i.h" 29 30 static unsigned int dbg_level; 31 #include "../libcxgbi.h" 32 33 #define DRV_MODULE_NAME "cxgb3i" 34 #define DRV_MODULE_DESC "Chelsio T3 iSCSI Driver" 35 #define DRV_MODULE_VERSION "2.0.1-ko" 36 #define DRV_MODULE_RELDATE "Apr. 2015" 37 38 static char version[] = 39 DRV_MODULE_DESC " " DRV_MODULE_NAME 40 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 41 42 MODULE_AUTHOR("Chelsio Communications, Inc."); 43 MODULE_DESCRIPTION(DRV_MODULE_DESC); 44 MODULE_VERSION(DRV_MODULE_VERSION); 45 MODULE_LICENSE("GPL"); 46 47 module_param(dbg_level, uint, 0644); 48 MODULE_PARM_DESC(dbg_level, "debug flag (default=0)"); 49 50 static int cxgb3i_rcv_win = 256 * 1024; 51 module_param(cxgb3i_rcv_win, int, 0644); 52 MODULE_PARM_DESC(cxgb3i_rcv_win, "TCP receive window in bytes (default=256KB)"); 53 54 static int cxgb3i_snd_win = 128 * 1024; 55 module_param(cxgb3i_snd_win, int, 0644); 56 MODULE_PARM_DESC(cxgb3i_snd_win, "TCP send window in bytes (default=128KB)"); 57 58 static int cxgb3i_rx_credit_thres = 10 * 1024; 59 module_param(cxgb3i_rx_credit_thres, int, 0644); 60 MODULE_PARM_DESC(cxgb3i_rx_credit_thres, 61 "RX credits return threshold in bytes (default=10KB)"); 62 63 static unsigned int cxgb3i_max_connect = 8 * 1024; 64 module_param(cxgb3i_max_connect, uint, 0644); 65 MODULE_PARM_DESC(cxgb3i_max_connect, "Max. # of connections (default=8092)"); 66 67 static unsigned int cxgb3i_sport_base = 20000; 68 module_param(cxgb3i_sport_base, uint, 0644); 69 MODULE_PARM_DESC(cxgb3i_sport_base, "starting port number (default=20000)"); 70 71 static void cxgb3i_dev_open(struct t3cdev *); 72 static void cxgb3i_dev_close(struct t3cdev *); 73 static void cxgb3i_dev_event_handler(struct t3cdev *, u32, u32); 74 75 static struct cxgb3_client t3_client = { 76 .name = DRV_MODULE_NAME, 77 .handlers = cxgb3i_cpl_handlers, 78 .add = cxgb3i_dev_open, 79 .remove = cxgb3i_dev_close, 80 .event_handler = cxgb3i_dev_event_handler, 81 }; 82 83 static struct scsi_host_template cxgb3i_host_template = { 84 .module = THIS_MODULE, 85 .name = DRV_MODULE_NAME, 86 .proc_name = DRV_MODULE_NAME, 87 .can_queue = CXGB3I_SCSI_HOST_QDEPTH, 88 .queuecommand = iscsi_queuecommand, 89 .change_queue_depth = scsi_change_queue_depth, 90 .sg_tablesize = SG_ALL, 91 .max_sectors = 0xFFFF, 92 .cmd_per_lun = ISCSI_DEF_CMD_PER_LUN, 93 .eh_timed_out = iscsi_eh_cmd_timed_out, 94 .eh_abort_handler = iscsi_eh_abort, 95 .eh_device_reset_handler = iscsi_eh_device_reset, 96 .eh_target_reset_handler = iscsi_eh_recover_target, 97 .target_alloc = iscsi_target_alloc, 98 .dma_boundary = PAGE_SIZE - 1, 99 .this_id = -1, 100 .track_queue_depth = 1, 101 }; 102 103 static struct iscsi_transport cxgb3i_iscsi_transport = { 104 .owner = THIS_MODULE, 105 .name = DRV_MODULE_NAME, 106 /* owner and name should be set already */ 107 .caps = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST 108 | CAP_DATADGST | CAP_DIGEST_OFFLOAD | 109 CAP_PADDING_OFFLOAD | CAP_TEXT_NEGO, 110 .attr_is_visible = cxgbi_attr_is_visible, 111 .get_host_param = cxgbi_get_host_param, 112 .set_host_param = cxgbi_set_host_param, 113 /* session management */ 114 .create_session = cxgbi_create_session, 115 .destroy_session = cxgbi_destroy_session, 116 .get_session_param = iscsi_session_get_param, 117 /* connection management */ 118 .create_conn = cxgbi_create_conn, 119 .bind_conn = cxgbi_bind_conn, 120 .destroy_conn = iscsi_tcp_conn_teardown, 121 .start_conn = iscsi_conn_start, 122 .stop_conn = iscsi_conn_stop, 123 .get_conn_param = iscsi_conn_get_param, 124 .set_param = cxgbi_set_conn_param, 125 .get_stats = cxgbi_get_conn_stats, 126 /* pdu xmit req from user space */ 127 .send_pdu = iscsi_conn_send_pdu, 128 /* task */ 129 .init_task = iscsi_tcp_task_init, 130 .xmit_task = iscsi_tcp_task_xmit, 131 .cleanup_task = cxgbi_cleanup_task, 132 /* pdu */ 133 .alloc_pdu = cxgbi_conn_alloc_pdu, 134 .init_pdu = cxgbi_conn_init_pdu, 135 .xmit_pdu = cxgbi_conn_xmit_pdu, 136 .parse_pdu_itt = cxgbi_parse_pdu_itt, 137 /* TCP connect/disconnect */ 138 .get_ep_param = cxgbi_get_ep_param, 139 .ep_connect = cxgbi_ep_connect, 140 .ep_poll = cxgbi_ep_poll, 141 .ep_disconnect = cxgbi_ep_disconnect, 142 /* Error recovery timeout call */ 143 .session_recovery_timedout = iscsi_session_recovery_timedout, 144 }; 145 146 static struct scsi_transport_template *cxgb3i_stt; 147 148 /* 149 * CPL (Chelsio Protocol Language) defines a message passing interface between 150 * the host driver and Chelsio asic. 151 * The section below implments CPLs that related to iscsi tcp connection 152 * open/close/abort and data send/receive. 153 */ 154 155 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion); 156 157 static void send_act_open_req(struct cxgbi_sock *csk, struct sk_buff *skb, 158 const struct l2t_entry *e) 159 { 160 unsigned int wscale = cxgbi_sock_compute_wscale(csk->rcv_win); 161 struct cpl_act_open_req *req = (struct cpl_act_open_req *)skb->head; 162 163 skb->priority = CPL_PRIORITY_SETUP; 164 165 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 166 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, csk->atid)); 167 req->local_port = csk->saddr.sin_port; 168 req->peer_port = csk->daddr.sin_port; 169 req->local_ip = csk->saddr.sin_addr.s_addr; 170 req->peer_ip = csk->daddr.sin_addr.s_addr; 171 172 req->opt0h = htonl(V_KEEP_ALIVE(1) | F_TCAM_BYPASS | 173 V_WND_SCALE(wscale) | V_MSS_IDX(csk->mss_idx) | 174 V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); 175 req->opt0l = htonl(V_ULP_MODE(ULP2_MODE_ISCSI) | 176 V_RCV_BUFSIZ(csk->rcv_win >> 10)); 177 178 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 179 "csk 0x%p,%u,0x%lx,%u, %pI4:%u-%pI4:%u, %u,%u,%u.\n", 180 csk, csk->state, csk->flags, csk->atid, 181 &req->local_ip, ntohs(req->local_port), 182 &req->peer_ip, ntohs(req->peer_port), 183 csk->mss_idx, e->idx, e->smt_idx); 184 185 l2t_send(csk->cdev->lldev, skb, csk->l2t); 186 } 187 188 static inline void act_open_arp_failure(struct t3cdev *dev, struct sk_buff *skb) 189 { 190 cxgbi_sock_act_open_req_arp_failure(NULL, skb); 191 } 192 193 /* 194 * CPL connection close request: host -> 195 * 196 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to 197 * the write queue (i.e., after any unsent txt data). 198 */ 199 static void send_close_req(struct cxgbi_sock *csk) 200 { 201 struct sk_buff *skb = csk->cpl_close; 202 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head; 203 unsigned int tid = csk->tid; 204 205 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 206 "csk 0x%p,%u,0x%lx,%u.\n", 207 csk, csk->state, csk->flags, csk->tid); 208 209 csk->cpl_close = NULL; 210 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 211 req->wr.wr_lo = htonl(V_WR_TID(tid)); 212 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 213 req->rsvd = htonl(csk->write_seq); 214 215 cxgbi_sock_skb_entail(csk, skb); 216 if (csk->state >= CTP_ESTABLISHED) 217 push_tx_frames(csk, 1); 218 } 219 220 /* 221 * CPL connection abort request: host -> 222 * 223 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs 224 * for the same connection and also that we do not try to send a message 225 * after the connection has closed. 226 */ 227 static void abort_arp_failure(struct t3cdev *tdev, struct sk_buff *skb) 228 { 229 struct cpl_abort_req *req = cplhdr(skb); 230 231 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 232 "t3dev 0x%p, tid %u, skb 0x%p.\n", 233 tdev, GET_TID(req), skb); 234 req->cmd = CPL_ABORT_NO_RST; 235 cxgb3_ofld_send(tdev, skb); 236 } 237 238 static void send_abort_req(struct cxgbi_sock *csk) 239 { 240 struct sk_buff *skb = csk->cpl_abort_req; 241 struct cpl_abort_req *req; 242 243 if (unlikely(csk->state == CTP_ABORTING || !skb)) 244 return; 245 cxgbi_sock_set_state(csk, CTP_ABORTING); 246 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_PENDING); 247 /* Purge the send queue so we don't send anything after an abort. */ 248 cxgbi_sock_purge_write_queue(csk); 249 250 csk->cpl_abort_req = NULL; 251 req = (struct cpl_abort_req *)skb->head; 252 skb->priority = CPL_PRIORITY_DATA; 253 set_arp_failure_handler(skb, abort_arp_failure); 254 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 255 req->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 256 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, csk->tid)); 257 req->rsvd0 = htonl(csk->snd_nxt); 258 req->rsvd1 = !cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT); 259 req->cmd = CPL_ABORT_SEND_RST; 260 261 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 262 "csk 0x%p,%u,0x%lx,%u, snd_nxt %u, 0x%x.\n", 263 csk, csk->state, csk->flags, csk->tid, csk->snd_nxt, 264 req->rsvd1); 265 266 l2t_send(csk->cdev->lldev, skb, csk->l2t); 267 } 268 269 /* 270 * CPL connection abort reply: host -> 271 * 272 * Send an ABORT_RPL message in response of the ABORT_REQ received. 273 */ 274 static void send_abort_rpl(struct cxgbi_sock *csk, int rst_status) 275 { 276 struct sk_buff *skb = csk->cpl_abort_rpl; 277 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head; 278 279 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 280 "csk 0x%p,%u,0x%lx,%u, status %d.\n", 281 csk, csk->state, csk->flags, csk->tid, rst_status); 282 283 csk->cpl_abort_rpl = NULL; 284 skb->priority = CPL_PRIORITY_DATA; 285 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 286 rpl->wr.wr_lo = htonl(V_WR_TID(csk->tid)); 287 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, csk->tid)); 288 rpl->cmd = rst_status; 289 cxgb3_ofld_send(csk->cdev->lldev, skb); 290 } 291 292 /* 293 * CPL connection rx data ack: host -> 294 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of 295 * credits sent. 296 */ 297 static u32 send_rx_credits(struct cxgbi_sock *csk, u32 credits) 298 { 299 struct sk_buff *skb; 300 struct cpl_rx_data_ack *req; 301 u32 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 302 303 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 304 "csk 0x%p,%u,0x%lx,%u, credit %u, dack %u.\n", 305 csk, csk->state, csk->flags, csk->tid, credits, dack); 306 307 skb = alloc_wr(sizeof(*req), 0, GFP_ATOMIC); 308 if (!skb) { 309 pr_info("csk 0x%p, credit %u, OOM.\n", csk, credits); 310 return 0; 311 } 312 req = (struct cpl_rx_data_ack *)skb->head; 313 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 314 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, csk->tid)); 315 req->credit_dack = htonl(F_RX_DACK_CHANGE | V_RX_DACK_MODE(1) | 316 V_RX_CREDITS(credits)); 317 skb->priority = CPL_PRIORITY_ACK; 318 cxgb3_ofld_send(csk->cdev->lldev, skb); 319 return credits; 320 } 321 322 /* 323 * CPL connection tx data: host -> 324 * 325 * Send iscsi PDU via TX_DATA CPL message. Returns the number of 326 * credits sent. 327 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of 328 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3). 329 */ 330 331 static unsigned int wrlen __read_mostly; 332 static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly; 333 334 static void init_wr_tab(unsigned int wr_len) 335 { 336 int i; 337 338 if (skb_wrs[1]) /* already initialized */ 339 return; 340 for (i = 1; i < SKB_WR_LIST_SIZE; i++) { 341 int sgl_len = (3 * i) / 2 + (i & 1); 342 343 sgl_len += 3; 344 skb_wrs[i] = (sgl_len <= wr_len 345 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1)); 346 } 347 wrlen = wr_len * 8; 348 } 349 350 static inline void make_tx_data_wr(struct cxgbi_sock *csk, struct sk_buff *skb, 351 int len, int req_completion) 352 { 353 struct tx_data_wr *req; 354 struct l2t_entry *l2t = csk->l2t; 355 356 skb_reset_transport_header(skb); 357 req = __skb_push(skb, sizeof(*req)); 358 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA) | 359 (req_completion ? F_WR_COMPL : 0)); 360 req->wr_lo = htonl(V_WR_TID(csk->tid)); 361 /* len includes the length of any HW ULP additions */ 362 req->len = htonl(len); 363 /* V_TX_ULP_SUBMODE sets both the mode and submode */ 364 req->flags = htonl(V_TX_ULP_SUBMODE(cxgbi_skcb_ulp_mode(skb)) | 365 V_TX_SHOVE((skb_peek(&csk->write_queue) ? 0 : 1))); 366 req->sndseq = htonl(csk->snd_nxt); 367 req->param = htonl(V_TX_PORT(l2t->smt_idx)); 368 369 if (!cxgbi_sock_flag(csk, CTPF_TX_DATA_SENT)) { 370 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | 371 V_TX_CPU_IDX(csk->rss_qid)); 372 /* sendbuffer is in units of 32KB. */ 373 req->param |= htonl(V_TX_SNDBUF(csk->snd_win >> 15)); 374 cxgbi_sock_set_flag(csk, CTPF_TX_DATA_SENT); 375 } 376 } 377 378 /** 379 * push_tx_frames -- start transmit 380 * @c3cn: the offloaded connection 381 * @req_completion: request wr_ack or not 382 * 383 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a 384 * connection's send queue and sends them on to T3. Must be called with the 385 * connection's lock held. Returns the amount of send buffer space that was 386 * freed as a result of sending queued data to T3. 387 */ 388 389 static void arp_failure_skb_discard(struct t3cdev *dev, struct sk_buff *skb) 390 { 391 kfree_skb(skb); 392 } 393 394 static int push_tx_frames(struct cxgbi_sock *csk, int req_completion) 395 { 396 int total_size = 0; 397 struct sk_buff *skb; 398 399 if (unlikely(csk->state < CTP_ESTABLISHED || 400 csk->state == CTP_CLOSE_WAIT_1 || csk->state >= CTP_ABORTING)) { 401 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 402 "csk 0x%p,%u,0x%lx,%u, in closing state.\n", 403 csk, csk->state, csk->flags, csk->tid); 404 return 0; 405 } 406 407 while (csk->wr_cred && (skb = skb_peek(&csk->write_queue)) != NULL) { 408 int len = skb->len; /* length before skb_push */ 409 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len); 410 int wrs_needed = skb_wrs[frags]; 411 412 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen) 413 wrs_needed = 1; 414 415 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1); 416 417 if (csk->wr_cred < wrs_needed) { 418 log_debug(1 << CXGBI_DBG_PDU_TX, 419 "csk 0x%p, skb len %u/%u, frag %u, wr %d<%u.\n", 420 csk, skb->len, skb->data_len, frags, 421 wrs_needed, csk->wr_cred); 422 break; 423 } 424 425 __skb_unlink(skb, &csk->write_queue); 426 skb->priority = CPL_PRIORITY_DATA; 427 skb->csum = wrs_needed; /* remember this until the WR_ACK */ 428 csk->wr_cred -= wrs_needed; 429 csk->wr_una_cred += wrs_needed; 430 cxgbi_sock_enqueue_wr(csk, skb); 431 432 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 433 "csk 0x%p, enqueue, skb len %u/%u, frag %u, wr %d, " 434 "left %u, unack %u.\n", 435 csk, skb->len, skb->data_len, frags, skb->csum, 436 csk->wr_cred, csk->wr_una_cred); 437 438 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_NEED_HDR))) { 439 if ((req_completion && 440 csk->wr_una_cred == wrs_needed) || 441 csk->wr_una_cred >= csk->wr_max_cred / 2) { 442 req_completion = 1; 443 csk->wr_una_cred = 0; 444 } 445 len += cxgbi_ulp_extra_len(cxgbi_skcb_ulp_mode(skb)); 446 make_tx_data_wr(csk, skb, len, req_completion); 447 csk->snd_nxt += len; 448 cxgbi_skcb_clear_flag(skb, SKCBF_TX_NEED_HDR); 449 } 450 total_size += skb->truesize; 451 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_TX, 452 "csk 0x%p, tid 0x%x, send skb 0x%p.\n", 453 csk, csk->tid, skb); 454 set_arp_failure_handler(skb, arp_failure_skb_discard); 455 l2t_send(csk->cdev->lldev, skb, csk->l2t); 456 } 457 return total_size; 458 } 459 460 /* 461 * Process a CPL_ACT_ESTABLISH message: -> host 462 * Updates connection state from an active establish CPL message. Runs with 463 * the connection lock held. 464 */ 465 466 static inline void free_atid(struct cxgbi_sock *csk) 467 { 468 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) { 469 cxgb3_free_atid(csk->cdev->lldev, csk->atid); 470 cxgbi_sock_clear_flag(csk, CTPF_HAS_ATID); 471 cxgbi_sock_put(csk); 472 } 473 } 474 475 static int do_act_establish(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 476 { 477 struct cxgbi_sock *csk = ctx; 478 struct cpl_act_establish *req = cplhdr(skb); 479 unsigned int tid = GET_TID(req); 480 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 481 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ 482 483 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 484 "atid 0x%x,tid 0x%x, csk 0x%p,%u,0x%lx, isn %u.\n", 485 atid, atid, csk, csk->state, csk->flags, rcv_isn); 486 487 cxgbi_sock_get(csk); 488 cxgbi_sock_set_flag(csk, CTPF_HAS_TID); 489 csk->tid = tid; 490 cxgb3_insert_tid(csk->cdev->lldev, &t3_client, csk, tid); 491 492 free_atid(csk); 493 494 csk->rss_qid = G_QNUM(ntohs(skb->csum)); 495 496 spin_lock_bh(&csk->lock); 497 if (csk->retry_timer.function) { 498 del_timer(&csk->retry_timer); 499 csk->retry_timer.function = NULL; 500 } 501 502 if (unlikely(csk->state != CTP_ACTIVE_OPEN)) 503 pr_info("csk 0x%p,%u,0x%lx,%u, got EST.\n", 504 csk, csk->state, csk->flags, csk->tid); 505 506 csk->copied_seq = csk->rcv_wup = csk->rcv_nxt = rcv_isn; 507 if (csk->rcv_win > (M_RCV_BUFSIZ << 10)) 508 csk->rcv_wup -= csk->rcv_win - (M_RCV_BUFSIZ << 10); 509 510 cxgbi_sock_established(csk, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 511 512 if (unlikely(cxgbi_sock_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED))) 513 /* upper layer has requested closing */ 514 send_abort_req(csk); 515 else { 516 if (skb_queue_len(&csk->write_queue)) 517 push_tx_frames(csk, 1); 518 cxgbi_conn_tx_open(csk); 519 } 520 521 spin_unlock_bh(&csk->lock); 522 __kfree_skb(skb); 523 return 0; 524 } 525 526 /* 527 * Process a CPL_ACT_OPEN_RPL message: -> host 528 * Handle active open failures. 529 */ 530 static int act_open_rpl_status_to_errno(int status) 531 { 532 switch (status) { 533 case CPL_ERR_CONN_RESET: 534 return -ECONNREFUSED; 535 case CPL_ERR_ARP_MISS: 536 return -EHOSTUNREACH; 537 case CPL_ERR_CONN_TIMEDOUT: 538 return -ETIMEDOUT; 539 case CPL_ERR_TCAM_FULL: 540 return -ENOMEM; 541 case CPL_ERR_CONN_EXIST: 542 return -EADDRINUSE; 543 default: 544 return -EIO; 545 } 546 } 547 548 static void act_open_retry_timer(struct timer_list *t) 549 { 550 struct cxgbi_sock *csk = from_timer(csk, t, retry_timer); 551 struct sk_buff *skb; 552 553 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 554 "csk 0x%p,%u,0x%lx,%u.\n", 555 csk, csk->state, csk->flags, csk->tid); 556 557 cxgbi_sock_get(csk); 558 spin_lock_bh(&csk->lock); 559 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_ATOMIC); 560 if (!skb) 561 cxgbi_sock_fail_act_open(csk, -ENOMEM); 562 else { 563 skb->sk = (struct sock *)csk; 564 set_arp_failure_handler(skb, act_open_arp_failure); 565 send_act_open_req(csk, skb, csk->l2t); 566 } 567 spin_unlock_bh(&csk->lock); 568 cxgbi_sock_put(csk); 569 } 570 571 static int do_act_open_rpl(struct t3cdev *tdev, struct sk_buff *skb, void *ctx) 572 { 573 struct cxgbi_sock *csk = ctx; 574 struct cpl_act_open_rpl *rpl = cplhdr(skb); 575 576 pr_info("csk 0x%p,%u,0x%lx,%u, status %u, %pI4:%u-%pI4:%u.\n", 577 csk, csk->state, csk->flags, csk->atid, rpl->status, 578 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 579 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 580 581 if (rpl->status != CPL_ERR_TCAM_FULL && 582 rpl->status != CPL_ERR_CONN_EXIST && 583 rpl->status != CPL_ERR_ARP_MISS) 584 cxgb3_queue_tid_release(tdev, GET_TID(rpl)); 585 586 cxgbi_sock_get(csk); 587 spin_lock_bh(&csk->lock); 588 if (rpl->status == CPL_ERR_CONN_EXIST && 589 csk->retry_timer.function != act_open_retry_timer) { 590 csk->retry_timer.function = act_open_retry_timer; 591 mod_timer(&csk->retry_timer, jiffies + HZ / 2); 592 } else 593 cxgbi_sock_fail_act_open(csk, 594 act_open_rpl_status_to_errno(rpl->status)); 595 596 spin_unlock_bh(&csk->lock); 597 cxgbi_sock_put(csk); 598 __kfree_skb(skb); 599 return 0; 600 } 601 602 /* 603 * Process PEER_CLOSE CPL messages: -> host 604 * Handle peer FIN. 605 */ 606 static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 607 { 608 struct cxgbi_sock *csk = ctx; 609 610 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 611 "csk 0x%p,%u,0x%lx,%u.\n", 612 csk, csk->state, csk->flags, csk->tid); 613 614 cxgbi_sock_rcv_peer_close(csk); 615 __kfree_skb(skb); 616 return 0; 617 } 618 619 /* 620 * Process CLOSE_CONN_RPL CPL message: -> host 621 * Process a peer ACK to our FIN. 622 */ 623 static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb, 624 void *ctx) 625 { 626 struct cxgbi_sock *csk = ctx; 627 struct cpl_close_con_rpl *rpl = cplhdr(skb); 628 629 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 630 "csk 0x%p,%u,0x%lx,%u, snxt %u.\n", 631 csk, csk->state, csk->flags, csk->tid, ntohl(rpl->snd_nxt)); 632 633 cxgbi_sock_rcv_close_conn_rpl(csk, ntohl(rpl->snd_nxt)); 634 __kfree_skb(skb); 635 return 0; 636 } 637 638 /* 639 * Process ABORT_REQ_RSS CPL message: -> host 640 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this 641 * request except that we need to reply to it. 642 */ 643 644 static int abort_status_to_errno(struct cxgbi_sock *csk, int abort_reason, 645 int *need_rst) 646 { 647 switch (abort_reason) { 648 case CPL_ERR_BAD_SYN: /* fall through */ 649 case CPL_ERR_CONN_RESET: 650 return csk->state > CTP_ESTABLISHED ? -EPIPE : -ECONNRESET; 651 case CPL_ERR_XMIT_TIMEDOUT: 652 case CPL_ERR_PERSIST_TIMEDOUT: 653 case CPL_ERR_FINWAIT2_TIMEDOUT: 654 case CPL_ERR_KEEPALIVE_TIMEDOUT: 655 return -ETIMEDOUT; 656 default: 657 return -EIO; 658 } 659 } 660 661 static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 662 { 663 const struct cpl_abort_req_rss *req = cplhdr(skb); 664 struct cxgbi_sock *csk = ctx; 665 int rst_status = CPL_ABORT_NO_RST; 666 667 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 668 "csk 0x%p,%u,0x%lx,%u.\n", 669 csk, csk->state, csk->flags, csk->tid); 670 671 if (req->status == CPL_ERR_RTX_NEG_ADVICE || 672 req->status == CPL_ERR_PERSIST_NEG_ADVICE) { 673 goto done; 674 } 675 676 cxgbi_sock_get(csk); 677 spin_lock_bh(&csk->lock); 678 679 if (!cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) { 680 cxgbi_sock_set_flag(csk, CTPF_ABORT_REQ_RCVD); 681 cxgbi_sock_set_state(csk, CTP_ABORTING); 682 goto out; 683 } 684 685 cxgbi_sock_clear_flag(csk, CTPF_ABORT_REQ_RCVD); 686 send_abort_rpl(csk, rst_status); 687 688 if (!cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 689 csk->err = abort_status_to_errno(csk, req->status, &rst_status); 690 cxgbi_sock_closed(csk); 691 } 692 693 out: 694 spin_unlock_bh(&csk->lock); 695 cxgbi_sock_put(csk); 696 done: 697 __kfree_skb(skb); 698 return 0; 699 } 700 701 /* 702 * Process ABORT_RPL_RSS CPL message: -> host 703 * Process abort replies. We only process these messages if we anticipate 704 * them as the coordination between SW and HW in this area is somewhat lacking 705 * and sometimes we get ABORT_RPLs after we are done with the connection that 706 * originated the ABORT_REQ. 707 */ 708 static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 709 { 710 struct cpl_abort_rpl_rss *rpl = cplhdr(skb); 711 struct cxgbi_sock *csk = ctx; 712 713 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 714 "status 0x%x, csk 0x%p, s %u, 0x%lx.\n", 715 rpl->status, csk, csk ? csk->state : 0, 716 csk ? csk->flags : 0UL); 717 /* 718 * Ignore replies to post-close aborts indicating that the abort was 719 * requested too late. These connections are terminated when we get 720 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss 721 * arrives the TID is either no longer used or it has been recycled. 722 */ 723 if (rpl->status == CPL_ERR_ABORT_FAILED) 724 goto rel_skb; 725 /* 726 * Sometimes we've already closed the connection, e.g., a post-close 727 * abort races with ABORT_REQ_RSS, the latter frees the connection 728 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, 729 * but FW turns the ABORT_REQ into a regular one and so we get 730 * ABORT_RPL_RSS with status 0 and no connection. 731 */ 732 if (csk) 733 cxgbi_sock_rcv_abort_rpl(csk); 734 rel_skb: 735 __kfree_skb(skb); 736 return 0; 737 } 738 739 /* 740 * Process RX_ISCSI_HDR CPL message: -> host 741 * Handle received PDUs, the payload could be DDP'ed. If not, the payload 742 * follow after the bhs. 743 */ 744 static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx) 745 { 746 struct cxgbi_sock *csk = ctx; 747 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb); 748 struct cpl_iscsi_hdr_norss data_cpl; 749 struct cpl_rx_data_ddp_norss ddp_cpl; 750 unsigned int hdr_len, data_len, status; 751 unsigned int len; 752 int err; 753 754 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 755 "csk 0x%p,%u,0x%lx,%u, skb 0x%p,%u.\n", 756 csk, csk->state, csk->flags, csk->tid, skb, skb->len); 757 758 spin_lock_bh(&csk->lock); 759 760 if (unlikely(csk->state >= CTP_PASSIVE_CLOSE)) { 761 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 762 "csk 0x%p,%u,0x%lx,%u, bad state.\n", 763 csk, csk->state, csk->flags, csk->tid); 764 if (csk->state != CTP_ABORTING) 765 goto abort_conn; 766 else 767 goto discard; 768 } 769 770 cxgbi_skcb_tcp_seq(skb) = ntohl(hdr_cpl->seq); 771 cxgbi_skcb_flags(skb) = 0; 772 773 skb_reset_transport_header(skb); 774 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr)); 775 776 len = hdr_len = ntohs(hdr_cpl->len); 777 /* msg coalesce is off or not enough data received */ 778 if (skb->len <= hdr_len) { 779 pr_err("%s: tid %u, CPL_ISCSI_HDR, skb len %u < %u.\n", 780 csk->cdev->ports[csk->port_id]->name, csk->tid, 781 skb->len, hdr_len); 782 goto abort_conn; 783 } 784 cxgbi_skcb_set_flag(skb, SKCBF_RX_COALESCED); 785 786 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl, 787 sizeof(ddp_cpl)); 788 if (err < 0) { 789 pr_err("%s: tid %u, copy cpl_ddp %u-%zu failed %d.\n", 790 csk->cdev->ports[csk->port_id]->name, csk->tid, 791 skb->len, sizeof(ddp_cpl), err); 792 goto abort_conn; 793 } 794 795 cxgbi_skcb_set_flag(skb, SKCBF_RX_STATUS); 796 cxgbi_skcb_rx_pdulen(skb) = ntohs(ddp_cpl.len); 797 cxgbi_skcb_rx_ddigest(skb) = ntohl(ddp_cpl.ulp_crc); 798 status = ntohl(ddp_cpl.ddp_status); 799 800 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 801 "csk 0x%p, skb 0x%p,%u, pdulen %u, status 0x%x.\n", 802 csk, skb, skb->len, cxgbi_skcb_rx_pdulen(skb), status); 803 804 if (status & (1 << CPL_RX_DDP_STATUS_HCRC_SHIFT)) 805 cxgbi_skcb_set_flag(skb, SKCBF_RX_HCRC_ERR); 806 if (status & (1 << CPL_RX_DDP_STATUS_DCRC_SHIFT)) 807 cxgbi_skcb_set_flag(skb, SKCBF_RX_DCRC_ERR); 808 if (status & (1 << CPL_RX_DDP_STATUS_PAD_SHIFT)) 809 cxgbi_skcb_set_flag(skb, SKCBF_RX_PAD_ERR); 810 811 if (skb->len > (hdr_len + sizeof(ddp_cpl))) { 812 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl)); 813 if (err < 0) { 814 pr_err("%s: tid %u, cp %zu/%u failed %d.\n", 815 csk->cdev->ports[csk->port_id]->name, 816 csk->tid, sizeof(data_cpl), skb->len, err); 817 goto abort_conn; 818 } 819 data_len = ntohs(data_cpl.len); 820 log_debug(1 << CXGBI_DBG_DDP | 1 << CXGBI_DBG_PDU_RX, 821 "skb 0x%p, pdu not ddp'ed %u/%u, status 0x%x.\n", 822 skb, data_len, cxgbi_skcb_rx_pdulen(skb), status); 823 len += sizeof(data_cpl) + data_len; 824 } else if (status & (1 << CPL_RX_DDP_STATUS_DDP_SHIFT)) 825 cxgbi_skcb_set_flag(skb, SKCBF_RX_DATA_DDPD); 826 827 csk->rcv_nxt = ntohl(ddp_cpl.seq) + cxgbi_skcb_rx_pdulen(skb); 828 __pskb_trim(skb, len); 829 __skb_queue_tail(&csk->receive_queue, skb); 830 cxgbi_conn_pdu_ready(csk); 831 832 spin_unlock_bh(&csk->lock); 833 return 0; 834 835 abort_conn: 836 send_abort_req(csk); 837 discard: 838 spin_unlock_bh(&csk->lock); 839 __kfree_skb(skb); 840 return 0; 841 } 842 843 /* 844 * Process TX_DATA_ACK CPL messages: -> host 845 * Process an acknowledgment of WR completion. Advance snd_una and send the 846 * next batch of work requests from the write queue. 847 */ 848 static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx) 849 { 850 struct cxgbi_sock *csk = ctx; 851 struct cpl_wr_ack *hdr = cplhdr(skb); 852 853 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_PDU_RX, 854 "csk 0x%p,%u,0x%lx,%u, cr %u.\n", 855 csk, csk->state, csk->flags, csk->tid, ntohs(hdr->credits)); 856 857 cxgbi_sock_rcv_wr_ack(csk, ntohs(hdr->credits), ntohl(hdr->snd_una), 1); 858 __kfree_skb(skb); 859 return 0; 860 } 861 862 /* 863 * for each connection, pre-allocate skbs needed for close/abort requests. So 864 * that we can service the request right away. 865 */ 866 static int alloc_cpls(struct cxgbi_sock *csk) 867 { 868 csk->cpl_close = alloc_wr(sizeof(struct cpl_close_con_req), 0, 869 GFP_KERNEL); 870 if (!csk->cpl_close) 871 return -ENOMEM; 872 csk->cpl_abort_req = alloc_wr(sizeof(struct cpl_abort_req), 0, 873 GFP_KERNEL); 874 if (!csk->cpl_abort_req) 875 goto free_cpl_skbs; 876 877 csk->cpl_abort_rpl = alloc_wr(sizeof(struct cpl_abort_rpl), 0, 878 GFP_KERNEL); 879 if (!csk->cpl_abort_rpl) 880 goto free_cpl_skbs; 881 882 return 0; 883 884 free_cpl_skbs: 885 cxgbi_sock_free_cpl_skbs(csk); 886 return -ENOMEM; 887 } 888 889 /** 890 * release_offload_resources - release offload resource 891 * @c3cn: the offloaded iscsi tcp connection. 892 * Release resources held by an offload connection (TID, L2T entry, etc.) 893 */ 894 static void l2t_put(struct cxgbi_sock *csk) 895 { 896 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 897 898 if (csk->l2t) { 899 l2t_release(t3dev, csk->l2t); 900 csk->l2t = NULL; 901 cxgbi_sock_put(csk); 902 } 903 } 904 905 static void release_offload_resources(struct cxgbi_sock *csk) 906 { 907 struct t3cdev *t3dev = (struct t3cdev *)csk->cdev->lldev; 908 909 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 910 "csk 0x%p,%u,0x%lx,%u.\n", 911 csk, csk->state, csk->flags, csk->tid); 912 913 csk->rss_qid = 0; 914 cxgbi_sock_free_cpl_skbs(csk); 915 916 if (csk->wr_cred != csk->wr_max_cred) { 917 cxgbi_sock_purge_wr_queue(csk); 918 cxgbi_sock_reset_wr_list(csk); 919 } 920 l2t_put(csk); 921 if (cxgbi_sock_flag(csk, CTPF_HAS_ATID)) 922 free_atid(csk); 923 else if (cxgbi_sock_flag(csk, CTPF_HAS_TID)) { 924 cxgb3_remove_tid(t3dev, (void *)csk, csk->tid); 925 cxgbi_sock_clear_flag(csk, CTPF_HAS_TID); 926 cxgbi_sock_put(csk); 927 } 928 csk->dst = NULL; 929 csk->cdev = NULL; 930 } 931 932 static void update_address(struct cxgbi_hba *chba) 933 { 934 if (chba->ipv4addr) { 935 if (chba->vdev && 936 chba->ipv4addr != cxgb3i_get_private_ipv4addr(chba->vdev)) { 937 cxgb3i_set_private_ipv4addr(chba->vdev, chba->ipv4addr); 938 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 939 pr_info("%s set %pI4.\n", 940 chba->vdev->name, &chba->ipv4addr); 941 } else if (chba->ipv4addr != 942 cxgb3i_get_private_ipv4addr(chba->ndev)) { 943 cxgb3i_set_private_ipv4addr(chba->ndev, chba->ipv4addr); 944 pr_info("%s set %pI4.\n", 945 chba->ndev->name, &chba->ipv4addr); 946 } 947 } else if (cxgb3i_get_private_ipv4addr(chba->ndev)) { 948 if (chba->vdev) 949 cxgb3i_set_private_ipv4addr(chba->vdev, 0); 950 cxgb3i_set_private_ipv4addr(chba->ndev, 0); 951 } 952 } 953 954 static int init_act_open(struct cxgbi_sock *csk) 955 { 956 struct dst_entry *dst = csk->dst; 957 struct cxgbi_device *cdev = csk->cdev; 958 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 959 struct net_device *ndev = cdev->ports[csk->port_id]; 960 struct cxgbi_hba *chba = cdev->hbas[csk->port_id]; 961 struct sk_buff *skb = NULL; 962 int ret; 963 964 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 965 "csk 0x%p,%u,0x%lx.\n", csk, csk->state, csk->flags); 966 967 update_address(chba); 968 if (chba->ipv4addr) 969 csk->saddr.sin_addr.s_addr = chba->ipv4addr; 970 971 csk->rss_qid = 0; 972 csk->l2t = t3_l2t_get(t3dev, dst, ndev, 973 &csk->daddr.sin_addr.s_addr); 974 if (!csk->l2t) { 975 pr_err("NO l2t available.\n"); 976 return -EINVAL; 977 } 978 cxgbi_sock_get(csk); 979 980 csk->atid = cxgb3_alloc_atid(t3dev, &t3_client, csk); 981 if (csk->atid < 0) { 982 pr_err("NO atid available.\n"); 983 ret = -EINVAL; 984 goto put_sock; 985 } 986 cxgbi_sock_set_flag(csk, CTPF_HAS_ATID); 987 cxgbi_sock_get(csk); 988 989 skb = alloc_wr(sizeof(struct cpl_act_open_req), 0, GFP_KERNEL); 990 if (!skb) { 991 ret = -ENOMEM; 992 goto free_atid; 993 } 994 skb->sk = (struct sock *)csk; 995 set_arp_failure_handler(skb, act_open_arp_failure); 996 csk->snd_win = cxgb3i_snd_win; 997 csk->rcv_win = cxgb3i_rcv_win; 998 999 csk->wr_max_cred = csk->wr_cred = T3C_DATA(t3dev)->max_wrs - 1; 1000 csk->wr_una_cred = 0; 1001 csk->mss_idx = cxgbi_sock_select_mss(csk, dst_mtu(dst)); 1002 cxgbi_sock_reset_wr_list(csk); 1003 csk->err = 0; 1004 1005 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1006 "csk 0x%p,%u,0x%lx, %pI4:%u-%pI4:%u.\n", 1007 csk, csk->state, csk->flags, 1008 &csk->saddr.sin_addr.s_addr, ntohs(csk->saddr.sin_port), 1009 &csk->daddr.sin_addr.s_addr, ntohs(csk->daddr.sin_port)); 1010 1011 cxgbi_sock_set_state(csk, CTP_ACTIVE_OPEN); 1012 send_act_open_req(csk, skb, csk->l2t); 1013 return 0; 1014 1015 free_atid: 1016 cxgb3_free_atid(t3dev, csk->atid); 1017 put_sock: 1018 cxgbi_sock_put(csk); 1019 l2t_release(t3dev, csk->l2t); 1020 csk->l2t = NULL; 1021 1022 return ret; 1023 } 1024 1025 cxgb3_cpl_handler_func cxgb3i_cpl_handlers[NUM_CPL_CMDS] = { 1026 [CPL_ACT_ESTABLISH] = do_act_establish, 1027 [CPL_ACT_OPEN_RPL] = do_act_open_rpl, 1028 [CPL_PEER_CLOSE] = do_peer_close, 1029 [CPL_ABORT_REQ_RSS] = do_abort_req, 1030 [CPL_ABORT_RPL_RSS] = do_abort_rpl, 1031 [CPL_CLOSE_CON_RPL] = do_close_con_rpl, 1032 [CPL_TX_DMA_ACK] = do_wr_ack, 1033 [CPL_ISCSI_HDR] = do_iscsi_hdr, 1034 }; 1035 1036 /** 1037 * cxgb3i_ofld_init - allocate and initialize resources for each adapter found 1038 * @cdev: cxgbi adapter 1039 */ 1040 static int cxgb3i_ofld_init(struct cxgbi_device *cdev) 1041 { 1042 struct t3cdev *t3dev = (struct t3cdev *)cdev->lldev; 1043 struct adap_ports port; 1044 struct ofld_page_info rx_page_info; 1045 unsigned int wr_len; 1046 int rc; 1047 1048 if (t3dev->ctl(t3dev, GET_WR_LEN, &wr_len) < 0 || 1049 t3dev->ctl(t3dev, GET_PORTS, &port) < 0 || 1050 t3dev->ctl(t3dev, GET_RX_PAGE_INFO, &rx_page_info) < 0) { 1051 pr_warn("t3 0x%p, offload up, ioctl failed.\n", t3dev); 1052 return -EINVAL; 1053 } 1054 1055 if (cxgb3i_max_connect > CXGBI_MAX_CONN) 1056 cxgb3i_max_connect = CXGBI_MAX_CONN; 1057 1058 rc = cxgbi_device_portmap_create(cdev, cxgb3i_sport_base, 1059 cxgb3i_max_connect); 1060 if (rc < 0) 1061 return rc; 1062 1063 init_wr_tab(wr_len); 1064 cdev->csk_release_offload_resources = release_offload_resources; 1065 cdev->csk_push_tx_frames = push_tx_frames; 1066 cdev->csk_send_abort_req = send_abort_req; 1067 cdev->csk_send_close_req = send_close_req; 1068 cdev->csk_send_rx_credits = send_rx_credits; 1069 cdev->csk_alloc_cpls = alloc_cpls; 1070 cdev->csk_init_act_open = init_act_open; 1071 1072 pr_info("cdev 0x%p, offload up, added.\n", cdev); 1073 return 0; 1074 } 1075 1076 /* 1077 * functions to program the pagepod in h/w 1078 */ 1079 static inline void ulp_mem_io_set_hdr(struct sk_buff *skb, unsigned int addr) 1080 { 1081 struct ulp_mem_io *req = (struct ulp_mem_io *)skb->head; 1082 1083 memset(req, 0, sizeof(*req)); 1084 1085 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 1086 req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(addr >> 5) | 1087 V_ULPTX_CMD(ULP_MEM_WRITE)); 1088 req->len = htonl(V_ULP_MEMIO_DATA_LEN(IPPOD_SIZE >> 5) | 1089 V_ULPTX_NFLITS((IPPOD_SIZE >> 3) + 1)); 1090 } 1091 1092 static struct cxgbi_ppm *cdev2ppm(struct cxgbi_device *cdev) 1093 { 1094 return ((struct t3cdev *)cdev->lldev)->ulp_iscsi; 1095 } 1096 1097 static int ddp_set_map(struct cxgbi_ppm *ppm, struct cxgbi_sock *csk, 1098 struct cxgbi_task_tag_info *ttinfo) 1099 { 1100 unsigned int idx = ttinfo->idx; 1101 unsigned int npods = ttinfo->npods; 1102 struct scatterlist *sg = ttinfo->sgl; 1103 struct cxgbi_pagepod *ppod; 1104 struct ulp_mem_io *req; 1105 unsigned int sg_off; 1106 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 1107 int i; 1108 1109 for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) { 1110 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + 1111 IPPOD_SIZE, 0, GFP_ATOMIC); 1112 1113 if (!skb) 1114 return -ENOMEM; 1115 ulp_mem_io_set_hdr(skb, pm_addr); 1116 req = (struct ulp_mem_io *)skb->head; 1117 ppod = (struct cxgbi_pagepod *)(req + 1); 1118 sg_off = i * PPOD_PAGES_MAX; 1119 cxgbi_ddp_set_one_ppod(ppod, ttinfo, &sg, 1120 &sg_off); 1121 skb->priority = CPL_PRIORITY_CONTROL; 1122 cxgb3_ofld_send(ppm->lldev, skb); 1123 } 1124 return 0; 1125 } 1126 1127 static void ddp_clear_map(struct cxgbi_device *cdev, struct cxgbi_ppm *ppm, 1128 struct cxgbi_task_tag_info *ttinfo) 1129 { 1130 unsigned int idx = ttinfo->idx; 1131 unsigned int pm_addr = (idx << PPOD_SIZE_SHIFT) + ppm->llimit; 1132 unsigned int npods = ttinfo->npods; 1133 int i; 1134 1135 log_debug(1 << CXGBI_DBG_DDP, 1136 "cdev 0x%p, clear idx %u, npods %u.\n", 1137 cdev, idx, npods); 1138 1139 for (i = 0; i < npods; i++, idx++, pm_addr += IPPOD_SIZE) { 1140 struct sk_buff *skb = alloc_wr(sizeof(struct ulp_mem_io) + 1141 IPPOD_SIZE, 0, GFP_ATOMIC); 1142 1143 if (!skb) { 1144 pr_err("cdev 0x%p, clear ddp, %u,%d/%u, skb OOM.\n", 1145 cdev, idx, i, npods); 1146 continue; 1147 } 1148 ulp_mem_io_set_hdr(skb, pm_addr); 1149 skb->priority = CPL_PRIORITY_CONTROL; 1150 cxgb3_ofld_send(ppm->lldev, skb); 1151 } 1152 } 1153 1154 static int ddp_setup_conn_pgidx(struct cxgbi_sock *csk, 1155 unsigned int tid, int pg_idx) 1156 { 1157 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1158 GFP_KERNEL); 1159 struct cpl_set_tcb_field *req; 1160 u64 val = pg_idx < DDP_PGIDX_MAX ? pg_idx : 0; 1161 1162 log_debug(1 << CXGBI_DBG_DDP, 1163 "csk 0x%p, tid %u, pg_idx %d.\n", csk, tid, pg_idx); 1164 if (!skb) 1165 return -ENOMEM; 1166 1167 /* set up ulp submode and page size */ 1168 req = (struct cpl_set_tcb_field *)skb->head; 1169 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1170 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1171 req->reply = V_NO_REPLY(1); 1172 req->cpu_idx = 0; 1173 req->word = htons(31); 1174 req->mask = cpu_to_be64(0xF0000000); 1175 req->val = cpu_to_be64(val << 28); 1176 skb->priority = CPL_PRIORITY_CONTROL; 1177 1178 cxgb3_ofld_send(csk->cdev->lldev, skb); 1179 return 0; 1180 } 1181 1182 /** 1183 * cxgb3i_setup_conn_digest - setup conn. digest setting 1184 * @csk: cxgb tcp socket 1185 * @tid: connection id 1186 * @hcrc: header digest enabled 1187 * @dcrc: data digest enabled 1188 * set up the iscsi digest settings for a connection identified by tid 1189 */ 1190 static int ddp_setup_conn_digest(struct cxgbi_sock *csk, unsigned int tid, 1191 int hcrc, int dcrc) 1192 { 1193 struct sk_buff *skb = alloc_wr(sizeof(struct cpl_set_tcb_field), 0, 1194 GFP_KERNEL); 1195 struct cpl_set_tcb_field *req; 1196 u64 val = (hcrc ? 1 : 0) | (dcrc ? 2 : 0); 1197 1198 log_debug(1 << CXGBI_DBG_DDP, 1199 "csk 0x%p, tid %u, crc %d,%d.\n", csk, tid, hcrc, dcrc); 1200 if (!skb) 1201 return -ENOMEM; 1202 1203 /* set up ulp submode and page size */ 1204 req = (struct cpl_set_tcb_field *)skb->head; 1205 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1206 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1207 req->reply = V_NO_REPLY(1); 1208 req->cpu_idx = 0; 1209 req->word = htons(31); 1210 req->mask = cpu_to_be64(0x0F000000); 1211 req->val = cpu_to_be64(val << 24); 1212 skb->priority = CPL_PRIORITY_CONTROL; 1213 1214 cxgb3_ofld_send(csk->cdev->lldev, skb); 1215 return 0; 1216 } 1217 1218 /** 1219 * cxgb3i_ddp_init - initialize the cxgb3 adapter's ddp resource 1220 * @cdev: cxgb3i adapter 1221 * initialize the ddp pagepod manager for a given adapter 1222 */ 1223 static int cxgb3i_ddp_init(struct cxgbi_device *cdev) 1224 { 1225 struct t3cdev *tdev = (struct t3cdev *)cdev->lldev; 1226 struct net_device *ndev = cdev->ports[0]; 1227 struct cxgbi_tag_format tformat; 1228 unsigned int ppmax, tagmask = 0; 1229 struct ulp_iscsi_info uinfo; 1230 int i, err; 1231 1232 err = tdev->ctl(tdev, ULP_ISCSI_GET_PARAMS, &uinfo); 1233 if (err < 0) { 1234 pr_err("%s, failed to get iscsi param %d.\n", 1235 ndev->name, err); 1236 return err; 1237 } 1238 if (uinfo.llimit >= uinfo.ulimit) { 1239 pr_warn("T3 %s, iscsi NOT enabled %u ~ %u!\n", 1240 ndev->name, uinfo.llimit, uinfo.ulimit); 1241 return -EACCES; 1242 } 1243 1244 ppmax = (uinfo.ulimit - uinfo.llimit + 1) >> PPOD_SIZE_SHIFT; 1245 tagmask = cxgbi_tagmask_set(ppmax); 1246 1247 pr_info("T3 %s: 0x%x~0x%x, 0x%x, tagmask 0x%x -> 0x%x.\n", 1248 ndev->name, uinfo.llimit, uinfo.ulimit, ppmax, uinfo.tagmask, 1249 tagmask); 1250 1251 memset(&tformat, 0, sizeof(struct cxgbi_tag_format)); 1252 for (i = 0; i < 4; i++) 1253 tformat.pgsz_order[i] = uinfo.pgsz_factor[i]; 1254 cxgbi_tagmask_check(tagmask, &tformat); 1255 1256 err = cxgbi_ddp_ppm_setup(&tdev->ulp_iscsi, cdev, &tformat, 1257 (uinfo.ulimit - uinfo.llimit + 1), 1258 uinfo.llimit, uinfo.llimit, 0, 0, 0); 1259 if (err) 1260 return err; 1261 1262 if (!(cdev->flags & CXGBI_FLAG_DDP_OFF)) { 1263 uinfo.tagmask = tagmask; 1264 uinfo.ulimit = uinfo.llimit + (ppmax << PPOD_SIZE_SHIFT); 1265 1266 err = tdev->ctl(tdev, ULP_ISCSI_SET_PARAMS, &uinfo); 1267 if (err < 0) { 1268 pr_err("T3 %s fail to set iscsi param %d.\n", 1269 ndev->name, err); 1270 cdev->flags |= CXGBI_FLAG_DDP_OFF; 1271 } 1272 err = 0; 1273 } 1274 1275 cdev->csk_ddp_setup_digest = ddp_setup_conn_digest; 1276 cdev->csk_ddp_setup_pgidx = ddp_setup_conn_pgidx; 1277 cdev->csk_ddp_set_map = ddp_set_map; 1278 cdev->csk_ddp_clear_map = ddp_clear_map; 1279 cdev->cdev2ppm = cdev2ppm; 1280 cdev->tx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1281 uinfo.max_txsz - ISCSI_PDU_NONPAYLOAD_LEN); 1282 cdev->rx_max_size = min_t(unsigned int, ULP2_MAX_PDU_PAYLOAD, 1283 uinfo.max_rxsz - ISCSI_PDU_NONPAYLOAD_LEN); 1284 1285 return 0; 1286 } 1287 1288 static void cxgb3i_dev_close(struct t3cdev *t3dev) 1289 { 1290 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1291 1292 if (!cdev || cdev->flags & CXGBI_FLAG_ADAPTER_RESET) { 1293 pr_info("0x%p close, f 0x%x.\n", cdev, cdev ? cdev->flags : 0); 1294 return; 1295 } 1296 1297 cxgbi_device_unregister(cdev); 1298 } 1299 1300 /** 1301 * cxgb3i_dev_open - init a t3 adapter structure and any h/w settings 1302 * @t3dev: t3cdev adapter 1303 */ 1304 static void cxgb3i_dev_open(struct t3cdev *t3dev) 1305 { 1306 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1307 struct adapter *adapter = tdev2adap(t3dev); 1308 int i, err; 1309 1310 if (cdev) { 1311 pr_info("0x%p, updating.\n", cdev); 1312 return; 1313 } 1314 1315 cdev = cxgbi_device_register(0, adapter->params.nports); 1316 if (!cdev) { 1317 pr_warn("device 0x%p register failed.\n", t3dev); 1318 return; 1319 } 1320 1321 cdev->flags = CXGBI_FLAG_DEV_T3 | CXGBI_FLAG_IPV4_SET; 1322 cdev->lldev = t3dev; 1323 cdev->pdev = adapter->pdev; 1324 cdev->ports = adapter->port; 1325 cdev->nports = adapter->params.nports; 1326 cdev->mtus = adapter->params.mtus; 1327 cdev->nmtus = NMTUS; 1328 cdev->rx_credit_thres = cxgb3i_rx_credit_thres; 1329 cdev->skb_tx_rsvd = CXGB3I_TX_HEADER_LEN; 1330 cdev->skb_rx_extra = sizeof(struct cpl_iscsi_hdr_norss); 1331 cdev->itp = &cxgb3i_iscsi_transport; 1332 1333 err = cxgb3i_ddp_init(cdev); 1334 if (err) { 1335 pr_info("0x%p ddp init failed %d\n", cdev, err); 1336 goto err_out; 1337 } 1338 1339 err = cxgb3i_ofld_init(cdev); 1340 if (err) { 1341 pr_info("0x%p offload init failed\n", cdev); 1342 goto err_out; 1343 } 1344 1345 err = cxgbi_hbas_add(cdev, CXGB3I_MAX_LUN, CXGBI_MAX_CONN, 1346 &cxgb3i_host_template, cxgb3i_stt); 1347 if (err) 1348 goto err_out; 1349 1350 for (i = 0; i < cdev->nports; i++) 1351 cdev->hbas[i]->ipv4addr = 1352 cxgb3i_get_private_ipv4addr(cdev->ports[i]); 1353 1354 pr_info("cdev 0x%p, f 0x%x, t3dev 0x%p open, err %d.\n", 1355 cdev, cdev ? cdev->flags : 0, t3dev, err); 1356 return; 1357 1358 err_out: 1359 cxgbi_device_unregister(cdev); 1360 } 1361 1362 static void cxgb3i_dev_event_handler(struct t3cdev *t3dev, u32 event, u32 port) 1363 { 1364 struct cxgbi_device *cdev = cxgbi_device_find_by_lldev(t3dev); 1365 1366 log_debug(1 << CXGBI_DBG_TOE, 1367 "0x%p, cdev 0x%p, event 0x%x, port 0x%x.\n", 1368 t3dev, cdev, event, port); 1369 if (!cdev) 1370 return; 1371 1372 switch (event) { 1373 case OFFLOAD_STATUS_DOWN: 1374 cdev->flags |= CXGBI_FLAG_ADAPTER_RESET; 1375 break; 1376 case OFFLOAD_STATUS_UP: 1377 cdev->flags &= ~CXGBI_FLAG_ADAPTER_RESET; 1378 break; 1379 } 1380 } 1381 1382 /** 1383 * cxgb3i_init_module - module init entry point 1384 * 1385 * initialize any driver wide global data structures and register itself 1386 * with the cxgb3 module 1387 */ 1388 static int __init cxgb3i_init_module(void) 1389 { 1390 int rc; 1391 1392 printk(KERN_INFO "%s", version); 1393 1394 rc = cxgbi_iscsi_init(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1395 if (rc < 0) 1396 return rc; 1397 1398 cxgb3_register_client(&t3_client); 1399 return 0; 1400 } 1401 1402 /** 1403 * cxgb3i_exit_module - module cleanup/exit entry point 1404 * 1405 * go through the driver hba list and for each hba, release any resource held. 1406 * and unregisters iscsi transport and the cxgb3 module 1407 */ 1408 static void __exit cxgb3i_exit_module(void) 1409 { 1410 cxgb3_unregister_client(&t3_client); 1411 cxgbi_device_unregister_all(CXGBI_FLAG_DEV_T3); 1412 cxgbi_iscsi_cleanup(&cxgb3i_iscsi_transport, &cxgb3i_stt); 1413 } 1414 1415 module_init(cxgb3i_init_module); 1416 module_exit(cxgb3i_exit_module); 1417